Commit 7a40c72f authored by Matthias's avatar Matthias

Merge branch 'MEMO-135-136-137-272-merge' of...

Merge branch 'MEMO-135-136-137-272-merge' of gitlab.switch.ch:memoriav/memobase-2020/services/import-process/import-api into MEMO-135-136-137-272-merge
parents 0bb8e43a 49c64ffa
Pipeline #10166 passed with stages
in 7 minutes and 14 seconds
......@@ -11,6 +11,9 @@ from import_api_app.resources.KafkaTopics import KafkaTopics
# from import_api_app.resources.ReadJobOptionsFromDrupal import ReadJobOptionsFromDrupal
import import_api_app.configuration
from import_api_app.resources.WriteJobResultsToDrupal import WriteJobResultToDrupal
from import_api_app.resources.ReadJobOptionsFromDrupal import ReadJobOptionsFromDrupal
# from import_api_app.resources.HelmStart import HelmStart
# from import_api_app.resources.HelmStop import HelmStop
import os
......@@ -62,9 +65,16 @@ def create_app(test_config=None):
'/v1/job/<institution_id>/<record_set_id>/<process_id>/<job_name>/start'
)
api.add_resource(JobStop, '/v1/job/<process_id>/<job_name>/stop')
# api.add_resource(
# HelmStart,
# '/v1/helm/<institution_id>/<record_set_id>/<process_id>/<job_name>/start'
# )
# api.add_resource(HelmStop, '/v1/helm/<process_id>/<job_name>/stop')
api.add_resource(Job, '/v1/job/<process_id>/<job_name>')
api.add_resource(WriteJobResultToDrupal, '/v1/drupal/<job_drupal_uuid>/<report>')
# api.add_resource(ReadJobOptionsFromDrupal, '/v1/fromDrupal/<job_drupal_uuid>')
api.add_resource(WriteJobResultToDrupal, '/v1/drupal/<job_log_drupal_uuid>/<report>')
api.add_resource(ReadJobOptionsFromDrupal, '/v1/drupal/<job_name>/<job_drupal_uuid>')
api.add_resource(KafkaTopics, '/v1/kafka/topics')
api.add_resource(JobReport, '/v1/job/<topic_name>/report')
......
......@@ -10,8 +10,8 @@ MAPPER_REGISTRY=cr.gitlab.switch.ch/memoriav/memobase-2020/services/import-proce
TFV_CHART_VERSION="0.3.5-chart"
TDT_CHART_VERSION="0.3.3-chart"
MAPPER_CHART_VERSION="0.6.5-chart"
TDT_CHART_VERSION="0.4.1-chart"
MAPPER_CHART_VERSION="0.8.0-chart"
helm chart pull ${TFV_REGISTRY}:${TFV_CHART_VERSION}
helm chart export ${TFV_REGISTRY}:${TFV_CHART_VERSION} -d charts/
......
from flask_restful import Resource, current_app, reqparse
from flasgger import swag_from
from import_api_app.utility import generate_helm_name
import import_api_app.helm as helm
import os
class HelmStart(Resource):
# Todo validate requests
# @swag.validate('job-parameters')
@swag_from('HelmStart.yml')
def post(self, institution_id, record_set_id, process_id, job_name):
parser = reqparse.RequestParser()
parser.add_argument('job-parameters', type=dict)
args = parser.parse_args()
job_parameters = args['job-parameters']
job_parameters['institutionId'] = institution_id
job_parameters['recordSetId'] = record_set_id
job_parameters['processId'] = process_id
helm_client = helm.Helm()
helm_name = generate_helm_name(process_id, job_name)
try:
output = helm_client.install(
chart=os.path.join(current_app.root_path, "charts", job_name),
name=helm_name,
namespace=current_app.config['NAMESPACE'],
set_values=job_parameters,
fail_on_err=False
)
message = (output.stdout + output.stderr)
message = message.replace('\n', '. ')
if output.returncode == 0:
return {
'status': 'SUCCESS',
'message': message,
'job_id': helm_name
}, 201
else:
return {
'status': 'FAILURE',
'message': message,
'job_id': helm_name
}, 500
except BaseException:
current_app.logger.error("It was not possible to run the helm install command")
return {'error': 'Unexpected Helm error'}, 500
Start a job in the kubernetes cluster (using helm charts)
---
tags:
- Helm
parameters:
- in: body
name: body
schema:
id: job-parameters
properties:
job-parameters:
type: object
properties:
appDirectory:
type: string
example: BECKER
description: The directory in the SFTP server
- in: path
name: institution_id
required: true
description: The Institution Identifier. We start with Drupal Node Id and will use the official identifier once it has been defined.
default: 123
type: string
- in: path
name: record_set_id
required: true
description: The ID of the Record Set (Bestand). We start with Drupal Node Id and will use the official identifier once it has been defined.
default: 456
type: string
- in: path
name: process_id
required: true
description: The identifier of the import process. Must be lowercase and
contain only letters, numbers and -.
default: 789
- in: path
type: string
name: job_name
example: text-file-validation
enum: ['text-file-validation', 'table-data-transform', 'mapper-service']
required: true
responses:
201:
description: Success, job is started
schema:
id: helm-result
properties:
status:
type: string
example: SUCCESS/FAILURE
enum: ['SUCCESS', 'FAILURE']
message:
type: string
example: 'NAME: j0001.
LAST DEPLOYED: Thu May 14 16:15:54 2020.
NAMESPACE: memobase. STATUS: deployed. REVISION: 1.
TEST SUITE: None.'
job_id:
type: string
example: j0001
500:
description: It was impossible to start the job
schema:
$ref: '#/definitions/helm-result'
from flask_restful import Resource, current_app
import import_api_app.helm as helm
from flasgger import swag_from
from import_api_app.utility import generate_helm_name
class HelmStop(Resource):
# Todo validate requests
# @swag.validate('job-parameters')
@swag_from('HelmStop.yml')
def delete(self, process_id, job_name):
helm_client = helm.Helm()
helm_name = generate_helm_name(process_id, job_name)
try:
output = helm_client.uninstall(
name=helm_name,
namespace=current_app.config['NAMESPACE'],
fail_on_err=False
)
message = output.stdout + output.stderr
message = message.replace('\n', '. ')
if output.returncode == 0:
return {
'status': 'SUCCESS',
'message': message,
'job_id': helm_name
}, 200
else:
return {
'status': 'FAILURE',
'message': message,
'job_id': helm_name
}, 500
except BaseException:
current_app.logger.error("It was not possible to run the helm install command")
return {'error': 'Unexpected Helm error'}, 500
Stop a job in the kubernetes cluster (using helm charts)
---
tags:
- Helm
parameters:
- in: path
name: process_id
required: true
description: The identifier of the import process. Must be lowercase and
contain only letters, numbers and -.
default: 789
- in: path
type: string
name: job_name
example: text-file-validation
enum: ['text-file-validation', 'table-data-transform', 'mapper-service']
required: true
responses:
200:
description: Success, job is deleted
schema:
id: helm-result
properties:
status:
type: string
example: SUCCESS/FAILURE
enum: ['SUCCESS', 'FAILURE']
message:
type: string
example: 'NAME: j0001. LAST DEPLOYED: Thu May 14 16:15 2020.
NAMESPACE: memobase. STATUS: deployed.
REVISION: 1. TEST SUITE: None.'
job_id:
type: string
example: j0001
500:
description: It was impossible to stop the job
schema:
$ref: '#/definitions/helm-result'
\ No newline at end of file
......@@ -20,11 +20,11 @@ class JobStart(Resource):
job_parameters['recordSetId'] = record_set_id
job_parameters['processId'] = process_id
helmClient = helm.Helm()
helm_client = helm.Helm()
helm_name = generate_helm_name(process_id, job_name)
try:
output = helmClient.install(
output = helm_client.install(
chart=os.path.join(current_app.root_path, "charts", job_name),
name=helm_name,
namespace=current_app.config['NAMESPACE'],
......
......@@ -60,8 +60,8 @@ responses:
properties:
status:
type: string
example: success/error
enum: ['success', 'error']
example: SUCCESS/FAILURE
enum: ['SUCCESS', 'FAILURE']
message:
type: string
example: 'NAME: j0001.
......
......@@ -9,12 +9,12 @@ class JobStop(Resource):
# @swag.validate('job-parameters')
@swag_from('JobStop.yml')
def delete(self, process_id, job_name):
helmClient = helm.Helm()
helm_client = helm.Helm()
helm_name = generate_helm_name(process_id, job_name)
try:
output = helmClient.uninstall(
output = helm_client.uninstall(
name=helm_name,
namespace=current_app.config['NAMESPACE'],
fail_on_err=False
......
......@@ -24,8 +24,8 @@ responses:
properties:
status:
type: string
example: success/error
enum: ['success', 'error']
example: SUCCESS/FAILURE
enum: ['SUCCESS', 'FAILURE']
message:
type: string
example: 'NAME: j0001. LAST DEPLOYED: Thu May 14 16:15 2020.
......
......@@ -90,7 +90,8 @@ class KafkaTopics(Resource):
new_topic_list = []
for topic in topics:
new_topic_list.append(NewTopic(name=topic, num_partitions=3, replication_factor=1))
# todo use multiple partitions
new_topic_list.append(NewTopic(name=topic, num_partitions=1, replication_factor=1))
try:
admin_client.create_topics(new_topic_list)
......
from flask_restful import Resource
from flask_restful import Resource, current_app
import jsonapi_requests
class ReadJobOptionsFromDrupal(Resource):
def get(self, job_drupal_uuid):
def get(self, job_name, job_drupal_uuid):
"""
Read the options from Drupal based on a job uuid
Read the options from Drupal based on a job uuid and transform it to camel case
---
tags:
- Drupal
parameters:
- in: path
type: string
name: job_name
default: table-data-transform
enum: ['text-file-validation', 'table-data-transform', 'mapper-service']
required: true
- in: path
type: string
name: job_drupal_uuid
description: The drupal uuid of the paragraph describing this job
default: 481790a3-8ba2-4fd7-95b4-c3a17f08d7d1
responses:
200:
description: The job data
schema:
id: jobs
properties:
job_id:
type: object
schema:
$ref: '#/definitions/job'
description: The job parameters
"""
api = jsonapi_requests.Api.config({
'API_ROOT': 'http://localhost:8080/jsonapi',
'API_ROOT': current_app.config['DRUPAL_API_URL'],
# 'AUTH': ('basic_auth_login', 'basic_auth_password'),
'VALIDATE_SSL': False,
'TIMEOUT': 1,
})
# in drupal job names have underscores instead of -
job_name = job_name.replace("-", "_")
endpoint = api.endpoint(
'paragraph/job_table_data_transform/481790a3-8ba2-4fd7-95b4-c3a17f08d7d1'
'paragraph/job_'+job_name+'/'+job_drupal_uuid
)
response = endpoint.get()
......
......@@ -4,7 +4,7 @@ import json
class WriteJobResultToDrupal(Resource):
def post(self, job_drupal_uuid, report):
def post(self, job_log_drupal_uuid, report):
"""
Write the job summary to Drupal job log (in field_summary)
---
......@@ -12,7 +12,7 @@ class WriteJobResultToDrupal(Resource):
- Drupal
parameters:
- in: path
name: job_drupal_uuid
name: job_log_drupal_uuid
required: true
description: job log uuid
default: 2b07341f-3b99-408c-a729-6e18dd77c6d5
......@@ -38,7 +38,7 @@ class WriteJobResultToDrupal(Resource):
data = {
"data": {
"id": job_drupal_uuid,
"id": job_log_drupal_uuid,
"type": "paragraph--job_log_result",
"attributes": {
"field_summary": report
......@@ -47,7 +47,7 @@ class WriteJobResultToDrupal(Resource):
}
url = current_app.config['DRUPAL_API_URL'] + \
'/paragraph/job_log_result/' + job_drupal_uuid
'/paragraph/job_log_result/' + job_log_drupal_uuid
try:
response = requests.patch(
......
......@@ -20,7 +20,7 @@ def test_job_start(client):
jobs_json = jobs_response.get_json()
number_of_jobs_before = len(jobs_json)
response = client.post('/v1/job/123/456/789/text-file-validation/start', json={
response = client.post('/v1/helm/123/456/789/text-file-validation/start', json={
"job-parameters": {
"appDirectory": "AFZ/AfZ-Becker-Audiovisuals"
}
......@@ -34,7 +34,7 @@ def test_job_start(client):
assert number_of_jobs_after == number_of_jobs_before + 1
# try to start a second job with same job id
response = client.post('/v1/job/123/456/789/text-file-validation/start', json={
response = client.post('/v1/helm/123/456/789/text-file-validation/start', json={
"job-parameters": {
"appDirectory": "AFZ/AfZ-Becker-Audiovisuals"
}
......@@ -42,7 +42,7 @@ def test_job_start(client):
assert response.status_code == 500
# use invalid chart name
response = client.post('/v1/job/123/456/789/TESTTEST/start', json={
response = client.post('/v1/helm/123/456/789/TESTTEST/start', json={
"job-parameters": {
"appDirectory": "AFZ/AfZ-Becker-Audiovisuals",
}
......@@ -59,13 +59,13 @@ def test_get_job(client):
def test_job_stop(client):
response = client.delete('/v1/job/789/text-file-validation/stop')
response = client.delete('/v1/helm/789/text-file-validation/stop')
assert response.status_code == 200
response = client.delete('/v1/job/789/jdskfhjkdshjkewhrjkewrjkewhjkhrjkewhr/stop')
response = client.delete('/v1/helm/789/jdskfhjkdshjkewhrjkewrjkewhjkhrjkewhr/stop')
assert response.status_code == 500
response = client.get('/v1/job/789/text-file-validation')
response = client.get('/v1/helm/789/text-file-validation')
assert response.status_code == 404
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment