Merge branch 'master' of gitlab.switch.ch:memoriav/memobase-2020/services/import-process/import-api

Dockerfile

@@ -17,7 +17,7 @@ WORKDIR /
 ADD setup.py /
 RUN pip install -e .
 ENV FLASK_APP import_api_app
-ENV FLASK_ENV development
+ENV FLASK_ENV production
 EXPOSE 5000
 ENTRYPOINT ["python"]
 CMD ["/import_api_app/main.py"]

import_api_app/pull-charts.sh

@@ -6,7 +6,7 @@ GITLAB_REGISTRY="cr.gitlab.switch.ch"
 
 TFV_REGISTRY=cr.gitlab.switch.ch/memoriav/memobase-2020/services/import-process/text-file-validation
 
-TFV_CHART_VERSION="1.0.15-chart"
+TFV_CHART_VERSION="1.0.17-chart"
 
 helm chart pull ${TFV_REGISTRY}:${TFV_CHART_VERSION}
 helm chart export ${TFV_REGISTRY}:${TFV_CHART_VERSION} -d charts/

import_api_app/resources/FetchMappingFile.py

@@ -49,10 +49,6 @@ class FetchMappingFile(Resource):
         port = current_app.config["sftp_port"]
         user = current_app.config["sftp_user"]
         pwd = current_app.config["sftp_password"]
-        # host = 'mb-wf2.memobase.unibas.ch'
-        # port = 80
-        # user = 'mb_sftp'
-        # pwd = '***'
         fileContentsMapping = ''
         fileContentsTransform = ''
         fileContentsLocalTransform = ''
@@ -88,7 +84,7 @@ class FetchMappingFile(Resource):
             raise ImportApiError(message)
 
         # 2. write file content into kafka topic
-        topic = 'import-process-config'
+        topic = 'mb-di-config-prod'
         try:
             producer = KafkaProducer(bootstrap_servers=current_app.config['kafka-broker-url'],
                                      key_serializer=str.encode)

import_api_app/resources/MonitorDrupalJsonApiRecordSet.py

@@ -81,7 +81,7 @@ class MonitorDrupalJsonApiRecordSet(Resource):
                                  'language_of_metadata/' + metadataLanguage['id'],
                                  headers=headers, auth=auth)
                 metadataLanguageCodes.append(
-                    drupalRecordSetLanguageCode.json()['data']['attributes']['langcode']
+                    drupalRecordSetLanguageCode.json()['data']['attributes']['name']
                 )
 
             field_related_record_sets = self.getRelatedRecordSets(drupalRecordSet)

import_api_app/resources/WriteJobResultsToDrupal.py

-from flask_restful import Resource, request, reqparse
+from flask_restful import Resource, request
 from flask_restful import current_app
 from helpers.Error import ImportApiError
 from datetime import datetime
@@ -83,29 +83,13 @@ class WriteJobResultToDrupal(Resource):
             description: There was a problem
         """
         try:
-            current_app.logger.debug("report data: " + bytes.decode(request.data))
-            parser = reqparse.RequestParser()
-            parser.add_argument('earliest')
-            parser.add_argument('elapsedTime')
-            parser.add_argument('fatal')
-            parser.add_argument('ignore')
-            parser.add_argument('institutionId')
-            parser.add_argument('latest')
-            parser.add_argument('messageId')
-            parser.add_argument('previousMessageId')
-            parser.add_argument('recordSetId')
-            parser.add_argument('sessionId')
-            parser.add_argument('step')
-            parser.add_argument('success')
-            parser.add_argument('total')
-            parser.add_argument('version')
-            parser.add_argument('warning')
-            args = parser.parse_args()
-            fatal = args['fatal']
+            body = request.json
+            report = json.dumps(body, indent=2)
+            current_app.logger.debug("report data: " + report)
+            fatal = body['fatal']
             status = 'FAILED'
             if fatal == 0:
                 status = 'SUCCESS'
-            report = json.dumps(request.json, indent=2)
             return write_results(job_drupal_uuid, job_log_drupal_uuid, status, report)
         except ImportApiError as e:
             return {'error': e.message}, 500

k8s-manifests/prod/import-api-prod-deployment.yml

 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: import-api-prod-deployment
+  name: api-import-prod
   namespace: memobase
   labels:
     app: import-api-prod-app