Merge branch 'MEMO-135-136-137-272-merge' of...

Merge branch 'MEMO-135-136-137-272-merge' of gitlab.switch.ch:memoriav/memobase-2020/services/import-process/import-api into MEMO-135-136-137-272-merge

import_api_app/__init__.py

@@ -11,6 +11,9 @@ from import_api_app.resources.KafkaTopics import KafkaTopics
 # from import_api_app.resources.ReadJobOptionsFromDrupal import ReadJobOptionsFromDrupal
 import import_api_app.configuration
 from import_api_app.resources.WriteJobResultsToDrupal import WriteJobResultToDrupal
+from import_api_app.resources.ReadJobOptionsFromDrupal import ReadJobOptionsFromDrupal
+# from import_api_app.resources.HelmStart import HelmStart
+# from import_api_app.resources.HelmStop import HelmStop
 import os
 
 
@@ -62,9 +65,16 @@ def create_app(test_config=None):
         '/v1/job/<institution_id>/<record_set_id>/<process_id>/<job_name>/start'
     )
     api.add_resource(JobStop, '/v1/job/<process_id>/<job_name>/stop')
+
+    # api.add_resource(
+    #    HelmStart,
+    #    '/v1/helm/<institution_id>/<record_set_id>/<process_id>/<job_name>/start'
+    # )
+    # api.add_resource(HelmStop, '/v1/helm/<process_id>/<job_name>/stop')
+
     api.add_resource(Job, '/v1/job/<process_id>/<job_name>')
-    api.add_resource(WriteJobResultToDrupal, '/v1/drupal/<job_drupal_uuid>/<report>')
-    # api.add_resource(ReadJobOptionsFromDrupal, '/v1/fromDrupal/<job_drupal_uuid>')
+    api.add_resource(WriteJobResultToDrupal, '/v1/drupal/<job_log_drupal_uuid>/<report>')
+    api.add_resource(ReadJobOptionsFromDrupal, '/v1/drupal/<job_name>/<job_drupal_uuid>')
 
     api.add_resource(KafkaTopics, '/v1/kafka/topics')
     api.add_resource(JobReport, '/v1/job/<topic_name>/report')

import_api_app/pull-charts.sh

@@ -10,8 +10,8 @@ MAPPER_REGISTRY=cr.gitlab.switch.ch/memoriav/memobase-2020/services/import-proce
 
 
 TFV_CHART_VERSION="0.3.5-chart"
-TDT_CHART_VERSION="0.3.3-chart"
-MAPPER_CHART_VERSION="0.6.5-chart"
+TDT_CHART_VERSION="0.4.1-chart"
+MAPPER_CHART_VERSION="0.8.0-chart"
 
 helm chart pull ${TFV_REGISTRY}:${TFV_CHART_VERSION}
 helm chart export ${TFV_REGISTRY}:${TFV_CHART_VERSION} -d charts/

import_api_app/resources/HelmStart.py

+from flask_restful import Resource, current_app, reqparse
+from flasgger import swag_from
+
+from import_api_app.utility import generate_helm_name
+import import_api_app.helm as helm
+import os
+
+
+class HelmStart(Resource):
+    # Todo validate requests
+    # @swag.validate('job-parameters')
+    @swag_from('HelmStart.yml')
+    def post(self, institution_id, record_set_id, process_id, job_name):
+
+        parser = reqparse.RequestParser()
+        parser.add_argument('job-parameters', type=dict)
+        args = parser.parse_args()
+        job_parameters = args['job-parameters']
+        job_parameters['institutionId'] = institution_id
+        job_parameters['recordSetId'] = record_set_id
+        job_parameters['processId'] = process_id
+
+        helm_client = helm.Helm()
+        helm_name = generate_helm_name(process_id, job_name)
+
+        try:
+            output = helm_client.install(
+                chart=os.path.join(current_app.root_path, "charts", job_name),
+                name=helm_name,
+                namespace=current_app.config['NAMESPACE'],
+                set_values=job_parameters,
+                fail_on_err=False
+            )
+            message = (output.stdout + output.stderr)
+            message = message.replace('\n', '. ')
+            if output.returncode == 0:
+                return {
+                    'status': 'SUCCESS',
+                    'message': message,
+                    'job_id': helm_name
+                }, 201
+            else:
+                return {
+                    'status': 'FAILURE',
+                    'message': message,
+                    'job_id': helm_name
+                }, 500
+        except BaseException:
+            current_app.logger.error("It was not possible to run the helm install command")
+            return {'error': 'Unexpected Helm error'}, 500

import_api_app/resources/HelmStart.yml

+Start a job in the kubernetes cluster (using helm charts)
+---
+tags:
+  - Helm
+parameters:
+  - in: body
+    name: body
+    schema:
+      id: job-parameters
+      properties:
+        job-parameters:
+          type: object
+          properties:
+            appDirectory:
+              type: string
+              example: BECKER
+              description: The directory in the SFTP server
+  - in: path
+    name: institution_id
+    required: true
+    description: The Institution Identifier. We start with Drupal Node Id and will use the official identifier once it has been defined.
+    default: 123
+    type: string
+  - in: path
+    name: record_set_id
+    required: true
+    description: The ID of the Record Set (Bestand). We start with Drupal Node Id and will use the official identifier once it has been defined.
+    default: 456
+    type: string
+  - in: path
+    name: process_id
+    required: true
+    description: The identifier of the import process. Must be lowercase and
+      contain only letters, numbers and -.
+    default: 789
+  - in: path
+    type: string
+    name: job_name
+    example: text-file-validation
+    enum: ['text-file-validation', 'table-data-transform', 'mapper-service']
+    required: true
+
+responses:
+  201:
+    description: Success, job is started
+    schema:
+      id: helm-result
+      properties:
+        status:
+          type: string
+          example: SUCCESS/FAILURE
+          enum: ['SUCCESS', 'FAILURE']
+        message:
+          type: string
+          example: 'NAME: j0001.
+             LAST DEPLOYED: Thu May 14 16:15:54 2020.
+             NAMESPACE: memobase. STATUS: deployed. REVISION: 1.
+             TEST SUITE: None.'
+        job_id:
+          type: string
+          example: j0001
+  500:
+    description: It was impossible to start the job
+    schema:
+     $ref: '#/definitions/helm-result'
+

import_api_app/resources/HelmStop.py

+from flask_restful import Resource, current_app
+import import_api_app.helm as helm
+from flasgger import swag_from
+from import_api_app.utility import generate_helm_name
+
+
+class HelmStop(Resource):
+    # Todo validate requests
+    # @swag.validate('job-parameters')
+    @swag_from('HelmStop.yml')
+    def delete(self, process_id, job_name):
+        helm_client = helm.Helm()
+
+        helm_name = generate_helm_name(process_id, job_name)
+
+        try:
+            output = helm_client.uninstall(
+                name=helm_name,
+                namespace=current_app.config['NAMESPACE'],
+                fail_on_err=False
+            )
+            message = output.stdout + output.stderr
+            message = message.replace('\n', '. ')
+            if output.returncode == 0:
+                return {
+                    'status': 'SUCCESS',
+                    'message': message,
+                    'job_id': helm_name
+                }, 200
+            else:
+                return {
+                    'status': 'FAILURE',
+                    'message': message,
+                    'job_id': helm_name
+                }, 500
+        except BaseException:
+            current_app.logger.error("It was not possible to run the helm install command")
+            return {'error': 'Unexpected Helm error'}, 500

import_api_app/resources/HelmStop.yml

+Stop a job in the kubernetes cluster (using helm charts)
+---
+tags:
+  - Helm
+parameters:
+  - in: path
+    name: process_id
+    required: true
+    description: The identifier of the import process. Must be lowercase and
+      contain only letters, numbers and -.
+    default: 789
+  - in: path
+    type: string
+    name: job_name
+    example: text-file-validation
+    enum: ['text-file-validation', 'table-data-transform', 'mapper-service']
+    required: true
+
+responses:
+  200:
+    description: Success, job is deleted
+    schema:
+      id: helm-result
+      properties:
+        status:
+          type: string
+          example: SUCCESS/FAILURE
+          enum: ['SUCCESS', 'FAILURE']
+        message:
+          type: string
+          example: 'NAME: j0001. LAST DEPLOYED: Thu May 14 16:15 2020.
+             NAMESPACE: memobase. STATUS: deployed.
+             REVISION: 1. TEST SUITE: None.'
+        job_id:
+          type: string
+          example: j0001
+  500:
+    description: It was impossible to stop the job
+    schema:
+     $ref: '#/definitions/helm-result'
\ No newline at end of file

import_api_app/resources/JobStart.py

@@ -20,11 +20,11 @@ class JobStart(Resource):
         job_parameters['recordSetId'] = record_set_id
         job_parameters['processId'] = process_id
 
-        helmClient = helm.Helm()
+        helm_client = helm.Helm()
         helm_name = generate_helm_name(process_id, job_name)
 
         try:
-            output = helmClient.install(
+            output = helm_client.install(
                 chart=os.path.join(current_app.root_path, "charts", job_name),
                 name=helm_name,
                 namespace=current_app.config['NAMESPACE'],

import_api_app/resources/JobStart.yml

@@ -60,8 +60,8 @@ responses:
       properties:
         status:
           type: string
-          example: success/error
-          enum: ['success', 'error']
+          example: SUCCESS/FAILURE
+          enum: ['SUCCESS', 'FAILURE']
         message:
           type: string
           example: 'NAME: j0001.

import_api_app/resources/JobStop.py

@@ -9,12 +9,12 @@ class JobStop(Resource):
     # @swag.validate('job-parameters')
     @swag_from('JobStop.yml')
     def delete(self, process_id, job_name):
-        helmClient = helm.Helm()
+        helm_client = helm.Helm()
 
         helm_name = generate_helm_name(process_id, job_name)
 
         try:
-            output = helmClient.uninstall(
+            output = helm_client.uninstall(
                 name=helm_name,
                 namespace=current_app.config['NAMESPACE'],
                 fail_on_err=False

import_api_app/resources/JobStop.yml

@@ -24,8 +24,8 @@ responses:
       properties:
         status:
           type: string
-          example: success/error
-          enum: ['success', 'error']
+          example: SUCCESS/FAILURE
+          enum: ['SUCCESS', 'FAILURE']
         message:
           type: string
           example: 'NAME: j0001. LAST DEPLOYED: Thu May 14 16:15 2020.

import_api_app/resources/KafkaTopics.py

@@ -90,7 +90,8 @@ class KafkaTopics(Resource):
         new_topic_list = []
 
         for topic in topics:
-            new_topic_list.append(NewTopic(name=topic, num_partitions=3, replication_factor=1))
+            # todo use multiple partitions
+            new_topic_list.append(NewTopic(name=topic, num_partitions=1, replication_factor=1))
 
         try:
             admin_client.create_topics(new_topic_list)

import_api_app/resources/ReadJobOptionsFromDrupal.py

-from flask_restful import Resource
+from flask_restful import Resource, current_app
 import jsonapi_requests
 
 
 class ReadJobOptionsFromDrupal(Resource):
-    def get(self, job_drupal_uuid):
+    def get(self, job_name, job_drupal_uuid):
         """
-        Read the options from Drupal based on a job uuid
+        Read the options from Drupal based on a job uuid and transform it to camel case
         ---
         tags:
           - Drupal
+        parameters:
+          - in: path
+            type: string
+            name: job_name
+            default: table-data-transform
+            enum: ['text-file-validation', 'table-data-transform', 'mapper-service']
+            required: true
+          - in: path
+            type: string
+            name: job_drupal_uuid
+            description: The drupal uuid of the paragraph describing this job
+            default: 481790a3-8ba2-4fd7-95b4-c3a17f08d7d1
         responses:
           200:
-            description: The job data
-            schema:
-              id: jobs
-              properties:
-                job_id:
-                  type: object
-                  schema:
-                    $ref: '#/definitions/job'
+            description: The job parameters
         """
 
         api = jsonapi_requests.Api.config({
-            'API_ROOT': 'http://localhost:8080/jsonapi',
+            'API_ROOT': current_app.config['DRUPAL_API_URL'],
             # 'AUTH': ('basic_auth_login', 'basic_auth_password'),
             'VALIDATE_SSL': False,
             'TIMEOUT': 1,
         })
 
+        # in drupal job names have underscores instead of -
+        job_name = job_name.replace("-", "_")
+
         endpoint = api.endpoint(
-            'paragraph/job_table_data_transform/481790a3-8ba2-4fd7-95b4-c3a17f08d7d1'
+            'paragraph/job_'+job_name+'/'+job_drupal_uuid
         )
         response = endpoint.get()
 

import_api_app/resources/WriteJobResultsToDrupal.py

@@ -4,7 +4,7 @@ import json
 
 
 class WriteJobResultToDrupal(Resource):
-    def post(self, job_drupal_uuid, report):
+    def post(self, job_log_drupal_uuid, report):
         """
         Write the job summary to Drupal job log (in field_summary)
         ---
@@ -12,7 +12,7 @@ class WriteJobResultToDrupal(Resource):
           - Drupal
         parameters:
         - in: path
-          name: job_drupal_uuid
+          name: job_log_drupal_uuid
           required: true
           description: job log uuid
           default: 2b07341f-3b99-408c-a729-6e18dd77c6d5
@@ -38,7 +38,7 @@ class WriteJobResultToDrupal(Resource):
 
         data = {
             "data": {
-                "id": job_drupal_uuid,
+                "id": job_log_drupal_uuid,
                 "type": "paragraph--job_log_result",
                 "attributes": {
                     "field_summary": report
@@ -47,7 +47,7 @@ class WriteJobResultToDrupal(Resource):
         }
 
         url = current_app.config['DRUPAL_API_URL'] + \
-            '/paragraph/job_log_result/' + job_drupal_uuid
+            '/paragraph/job_log_result/' + job_log_drupal_uuid
 
         try:
             response = requests.patch(

tests/test_import-api.py

@@ -20,7 +20,7 @@ def test_job_start(client):
     jobs_json = jobs_response.get_json()
     number_of_jobs_before = len(jobs_json)
 
-    response = client.post('/v1/job/123/456/789/text-file-validation/start', json={
+    response = client.post('/v1/helm/123/456/789/text-file-validation/start', json={
         "job-parameters": {
             "appDirectory": "AFZ/AfZ-Becker-Audiovisuals"
         }
@@ -34,7 +34,7 @@ def test_job_start(client):
     assert number_of_jobs_after == number_of_jobs_before + 1
 
     # try to start a second job with same job id
-    response = client.post('/v1/job/123/456/789/text-file-validation/start', json={
+    response = client.post('/v1/helm/123/456/789/text-file-validation/start', json={
         "job-parameters": {
             "appDirectory": "AFZ/AfZ-Becker-Audiovisuals"
         }
@@ -42,7 +42,7 @@ def test_job_start(client):
     assert response.status_code == 500
 
     # use invalid chart name
-    response = client.post('/v1/job/123/456/789/TESTTEST/start', json={
+    response = client.post('/v1/helm/123/456/789/TESTTEST/start', json={
         "job-parameters": {
             "appDirectory": "AFZ/AfZ-Becker-Audiovisuals",
         }
@@ -59,13 +59,13 @@ def test_get_job(client):
 
 
 def test_job_stop(client):
-    response = client.delete('/v1/job/789/text-file-validation/stop')
+    response = client.delete('/v1/helm/789/text-file-validation/stop')
     assert response.status_code == 200
 
-    response = client.delete('/v1/job/789/jdskfhjkdshjkewhrjkewrjkewhjkhrjkewhr/stop')
+    response = client.delete('/v1/helm/789/jdskfhjkdshjkewhrjkewrjkewhjkhrjkewhr/stop')
     assert response.status_code == 500
 
-    response = client.get('/v1/job/789/text-file-validation')
+    response = client.get('/v1/helm/789/text-file-validation')
     assert response.status_code == 404