Merge branch 'master' of gitlab.switch.ch:memoriav/memobase-2020/services/import-process/import-api

helm-charts/configmap/prod/gunicorn.conf.py

@@ -4,7 +4,7 @@ accesslog = "/dev/stdout"
 access_log_format = '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"'
 disable_redirect_access_to_syslog = True
 errorlog = "/dev/stderr"
-loglevel = 'info'
+loglevel = "info"
 workers = 1
 logconfig_dict = dict(
     version=1,
@@ -14,14 +14,14 @@ logconfig_dict = dict(
             "level": "INFO",
             "handlers": ["error_console"],
             "propagate": False,
-                "qualname": "gunicorn.error"
+            "qualname": "gunicorn.error",
         }
     },
     formatters={
         "generic": {
             "format": "[%(levelname)s] [%(name)s] %(message)s",
             "datefmt": "[%Y-%m-%d %H:%M:%S %z]",
-                "class": "logging.Formatter"
-            }
+            "class": "logging.Formatter",
         }
+    },
 )

helm-charts/configmap/test/gunicorn.conf.py

@@ -4,7 +4,7 @@ accesslog = "/dev/stdout"
 access_log_format = '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"'
 disable_redirect_access_to_syslog = True
 errorlog = "/dev/stderr"
-loglevel = 'debug'
+loglevel = "debug"
 workers = 1
 logconfig_dict = dict(
     version=1,
@@ -14,14 +14,14 @@ logconfig_dict = dict(
             "level": "DEBUG",
             "handlers": ["error_console"],
             "propagate": False,
-                "qualname": "gunicorn.error"
+            "qualname": "gunicorn.error",
         }
     },
     formatters={
         "generic": {
             "format": "[%(levelname)s] [%(name)s] %(message)s",
             "datefmt": "[%Y-%m-%d %H:%M:%S %z]",
-                "class": "logging.Formatter"
-            }
+            "class": "logging.Formatter",
         }
+    },
 )

import_api_app/helm.py

@@ -27,7 +27,7 @@ class Helm:
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
             check=fail_on_err,
-            text=True
+            text=True,
         )
 
     def install(

import_api_app/helpers/delete_service.py

@@ -20,7 +20,6 @@ from import_api_app.helpers.error import ImportApiError
 
 
 class DeleteService:
-
     def __init__(self, logger, namespace, root_path):
         self.namespace = namespace
         self.root_path = root_path
@@ -29,12 +28,12 @@ class DeleteService:
 
     def delete_record(self, record_id, session_id, dryrun):
         set_values = {
-            'sessionId': session_id,
-            'deleteObject': 'record',
-            'deleteId': record_id
+            "sessionId": session_id,
+            "deleteObject": "record",
+            "deleteId": record_id,
         }
         try:
-            return self.do_helm_install(set_values, dryrun)
+            return self.run_job(set_values, dryrun == "true")
         except Exception as ex:
             message = str(ex)
             self.logger.error(message)
@@ -42,12 +41,12 @@ class DeleteService:
 
     def delete_recordset(self, recordset_id, session_id, dryrun):
         set_values = {
-            'sessionId': session_id,
-            'deleteObject': 'record-set',
-            'deleteId': recordset_id
+            "sessionId": session_id,
+            "deleteObject": "record-set",
+            "deleteId": recordset_id,
         }
         try:
-            return self.do_helm_install(set_values, dryrun)
+            return self.run_job(set_values, dryrun == "true")
         except Exception as ex:
             message = str(ex)
             self.logger.error(message)
@@ -55,37 +54,72 @@ class DeleteService:
 
     def delete_institution(self, institution_id, session_id, dryrun):
         set_values = {
-            'sessionId': session_id,
-            'deleteObject': 'institution',
-            'deleteId': institution_id,
+            "sessionId": session_id,
+            "deleteObject": "institution",
+            "deleteId": institution_id,
         }
         try:
-            return self.do_helm_install(set_values, dryrun)
+            return self.run_job(set_values, dryrun == "true")
         except Exception as ex:
             message = str(ex)
             self.logger.error(message)
             raise ImportApiError(message)
 
-    def do_helm_install(self, set_values, dryrun):
+    def run_job(self, set_values: dict, dryrun: bool) -> str:
+        chart_name = "dd-marker-{}-{}".format(
+            os.getenv("ENV", ""), set_values["sessionId"]
+        )
+        res_install = self.do_helm_install(set_values, chart_name, dryrun)
+        if res_install.returncode > 0:
             self.logger.error(
-            'calling delete service: type=' +
-            set_values['deleteObject'] +
-            ' / id=' + set_values['deleteId'] +
-            ' / session=' + set_values['sessionId'])
+                "Something went when installing helm chart {}: returncode: {}; stdout: {}; stderr: {}".format(
+                    chart_name,
+                    res_install.returncode,
+                    res_install.stdout,
+                    res_install.stderr,
+                )
+            )
+        retJson = {
+            "returncode": res_install.returncode,
+            "stdout": res_install.stdout,
+            "stderr": res_install.stderr,
+        }
+        return json.dumps(retJson, ensure_ascii=True, indent=2)
+
+    def do_helm_install(self, set_values: dict, chart_name: str, dryrun: bool):
+        self.logger.error(
+            "calling delete service: type="
+            + set_values["deleteObject"]
+            + " / id="
+            + set_values["deleteId"]
+            + " / session="
+            + set_values["sessionId"]
+        )
+        # Due to limitations on arrays in helm argument values, we need to build a string
+        # (which will afterwards be converted back to an array by the k8s template)
         job_args = [
-            '--' + set_values['deleteObject'] + '-filter',
-            set_values['deleteId'],
-            set_values['sessionId']
+            "--" + set_values["deleteObject"] + "-filter",
+            set_values["deleteId"],
         ]
         if dryrun:
-            job_args.append('--dry-run')
-        job_args = {'jobArgs': job_args}
-        cp = self.helm.install(  # noqa: F841
-            chart=os.path.join(self.root_path, "charts", 'dd-marker-prod'),
-            name=set_values['sessionId'] + '-deleter',
+            job_args.extend(["--dry-run", set_values["sessionId"]])
+        else:
+            job_args.append(set_values["sessionId"])
+        # The sessionId is additionally used to build a distinct job id
+        job_args = {
+            "jobArgs": "'" + "' '".join(job_args) + "'",
+            "sessionId": set_values["sessionId"],
+        }
+        path_to_charts = os.path.join(self.root_path, "charts", "deletion-marker")
+        return self.helm.install(  # noqa: F841
+            chart=path_to_charts,
+            values_file=os.path.join(
+                path_to_charts,
+                "helm-values",
+                "dd-marker-" + os.getenv("ENV", "") + ".yaml",
+            ),
+            name=chart_name,
             namespace=self.namespace,
             set_values=job_args,
-            fail_on_err=False
+            fail_on_err=False,
         )
-        retJson = {'returncode': cp.returncode, 'stdout': cp.stdout, 'stderr': cp.stderr}
-        return json.dumps(retJson, ensure_ascii=True, indent=2)

import_api_app/helpers/mapping.py

@@ -20,13 +20,25 @@ from typing import Dict, Union
 import paramiko
 from kafka import KafkaProducer
 from kafka.errors import KafkaTimeoutError
-from paramiko.ssh_exception import AuthenticationException, BadHostKeyException, SSHException
+from paramiko.ssh_exception import (
+    AuthenticationException,
+    BadHostKeyException,
+    SSHException,
+)
 
 
 class MappingFileHandler:
-
-    def __init__(self, host: str, port: int, user: str, password: str, base_path: str,
-                 kafka_broker_url: str, topic: str, logger):
+    def __init__(
+        self,
+        host: str,
+        port: int,
+        user: str,
+        password: str,
+        base_path: str,
+        kafka_broker_url: str,
+        topic: str,
+        logger,
+    ):
         self.host = host
         self.port = port
         self.user = user
@@ -34,103 +46,104 @@ class MappingFileHandler:
         self.base_path = base_path
         self.logger = logger
 
-        self.config_folder = 'config'
-        self.mapping_file = 'mapping.yml'
-        self.xslt_file = 'transform.xslt'
-        self.local_transform_file = 'localTransforms.yml'
+        self.config_folder = "config"
+        self.mapping_file = "mapping.yml"
+        self.xslt_file = "transform.xslt"
+        self.local_transform_file = "localTransforms.yml"
         self.ssh_client = paramiko.SSHClient()
         self.ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
 
-        self.producer = KafkaProducer(bootstrap_servers=kafka_broker_url,
-                                      key_serializer=str.encode)
+        self.producer = KafkaProducer(
+            bootstrap_servers=kafka_broker_url, key_serializer=str.encode
+        )
         self.topic = topic
 
-    def process_mapping_files(self, record_set_id: str) -> Dict[str, Union[str, bool, list]]:
+    def process_mapping_files(
+        self, record_set_id: str
+    ) -> Dict[str, Union[str, bool, list]]:
         result = self._retrieve_mapping_files(record_set_id)
 
-        if result['status'] == 'FATAL':
-            result['mapping_file'] = False
-            result['xslt_file'] = False
-            result['local_transform_file'] = False
+        if result["status"] == "FATAL":
+            result["mapping_file"] = False
+            result["xslt_file"] = False
+            result["local_transform_file"] = False
             return result
 
         status = dict()
-        status['messages'] = list()
-        result_mapping = self._send_config_to_kafka(record_set_id, self.topic, 'mapping',
-                                                    result['mapping_file'])
-        if result_mapping['status'] == 'SUCCESS':
-            status['mapping_file'] = True
+        status["messages"] = list()
+        result_mapping = self._send_config_to_kafka(
+            record_set_id, self.topic, "mapping", result["mapping_file"]
+        )
+        if result_mapping["status"] == "SUCCESS":
+            status["mapping_file"] = True
         else:
-            status['mapping_file'] = False
-            status['status'] = 'FATAL'
-            status['messages'].append(result_mapping['message'])
-        if result['xslt_file'] != '':
-            result_xslt = self._send_config_to_kafka(record_set_id, self.topic, 'transform',
-                                                     result['xslt_file'])
-            if result_xslt['status'] == 'SUCCESS':
-                status['xslt_file'] = True
+            status["mapping_file"] = False
+            status["status"] = "FATAL"
+            status["messages"].append(result_mapping["message"])
+        if result["xslt_file"] != "":
+            result_xslt = self._send_config_to_kafka(
+                record_set_id, self.topic, "transform", result["xslt_file"]
+            )
+            if result_xslt["status"] == "SUCCESS":
+                status["xslt_file"] = True
             else:
-                status['xslt_file'] = False
-                status['status'] = 'WARNING'
-                status['messages'].append(result_xslt['message'])
+                status["xslt_file"] = False
+                status["status"] = "WARNING"
+                status["messages"].append(result_xslt["message"])
         else:
-            status['xslt_file'] = False
-            status['status'] = 'WARNING'
-        if result['local_transform'] != '':
-            result_local_transform = self._send_config_to_kafka(record_set_id, self.topic,
-                                                                'localTransform',
-                                                                result['local_transform'])
-            if result_local_transform['status'] == 'SUCCESS':
-                status['local_transform_file'] = True
+            status["xslt_file"] = False
+            status["status"] = "WARNING"
+        if result["local_transform"] != "":
+            result_local_transform = self._send_config_to_kafka(
+                record_set_id, self.topic, "localTransform", result["local_transform"]
+            )
+            if result_local_transform["status"] == "SUCCESS":
+                status["local_transform_file"] = True
             else:
-                status['local_transform_file'] = False
-                status['status'] = 'WARNING'
-                status['messages'].append(result_local_transform['message'])
+                status["local_transform_file"] = False
+                status["status"] = "WARNING"
+                status["messages"].append(result_local_transform["message"])
         else:
-            status['local_transform_file'] = False
-            status['status'] = 'WARNING'
-        if 'status' not in status:
-            status['status'] = 'SUCCESS'
+            status["local_transform_file"] = False
+            status["status"] = "WARNING"
+        if "status" not in status:
+            status["status"] = "SUCCESS"
         return status
 
-    def _retrieve_mapping_files(self, record_set_id: str) -> Dict[str, Union[str, bool, list]]:
+    def _retrieve_mapping_files(
+        self, record_set_id: str
+    ) -> Dict[str, Union[str, bool, list]]:
         try:
-            self.ssh_client.connect(hostname=self.host,
+            self.ssh_client.connect(
+                hostname=self.host,
                 port=self.port,
                 username=self.user,
-                                    password=self.password)
+                password=self.password,
+            )
         except BadHostKeyException as ex:
             message = f"Could not connect to the server because of a bad host key: {ex}"
             self.logger.error(message)
             return {
-                'status': 'FATAL',
-                'type': 'BadHostKeyException',
-                'messages': [message]
+                "status": "FATAL",
+                "type": "BadHostKeyException",
+                "messages": [message],
             }
         except AuthenticationException:
             message = "Could not authenticate with the sftp server with the given credentials."
             self.logger.error(message)
             return {
-                'status': 'FATAL',
-                'type': 'AuthenticationException',
-                'messages': [message]
+                "status": "FATAL",
+                "type": "AuthenticationException",
+                "messages": [message],
             }
         except SSHException as ex:
             message = f"SSH Exception: {ex}."
             self.logger.error(message)
-            return {
-                'status': 'FATAL',
-                'type': 'SSHException',
-                'messages': [message]
-            }
+            return {"status": "FATAL", "type": "SSHException", "messages": [message]}
         except socket.error as ex:
             message = f"Socket Error: {ex}."
             self.logger.error(message)
-            return {
-                'status': 'FATAL',
-                'type': 'SocketError',
-                'messages': [message]
-            }
+            return {"status": "FATAL", "type": "SocketError", "messages": [message]}
 
         sftp = self.ssh_client.open_sftp()
         path = join(self.base_path, record_set_id, self.config_folder)
@@ -138,60 +151,52 @@ class MappingFileHandler:
         # Check mapping file
         mapping_file_path = join(path, self.mapping_file)
         try:
-            with sftp.open(mapping_file_path, 'r') as mapping:
+            with sftp.open(mapping_file_path, "r") as mapping:
                 file_contents_mapping = mapping.read()
         except IOError as err:
-            message = f'Could not open the mapping file at {mapping_file_path} (ERROR: {err}).'
+            message = f"Could not open the mapping file at {mapping_file_path} (ERROR: {err})."
             sftp.close()
             self.ssh_client.close()
             self.logger.error(message)
-            return {
-                'status': 'FATAL',
-                'type': 'NoMappingFile',
-                'message': message
-            }
+            return {"status": "FATAL", "type": "NoMappingFile", "message": message}
 
         # Check xslt file
         xslt_file_path = join(path, self.xslt_file)
         try:
-            with sftp.open(xslt_file_path, 'r') as xslt:
+            with sftp.open(xslt_file_path, "r") as xslt:
                 file_contents_xslt = xslt.read()
         except IOError:
-            file_contents_xslt = ''
+            file_contents_xslt = ""
 
         # Check local transform file
         local_transform_file_path = join(path, self.local_transform_file)
         try:
-            with sftp.open(local_transform_file_path, 'r') as local_transform:
+            with sftp.open(local_transform_file_path, "r") as local_transform:
                 file_contents_local_transform = local_transform.read()
         except IOError:
-            file_contents_local_transform = ''
+            file_contents_local_transform = ""
 
         sftp.close()
         self.ssh_client.close()
         return {
-            'status': 'SUCCESS',
-            'mapping_file': file_contents_mapping,
-            'xslt_file': file_contents_xslt,
-            'local_transform': file_contents_local_transform
+            "status": "SUCCESS",
+            "mapping_file": file_contents_mapping,
+            "xslt_file": file_contents_xslt,
+            "local_transform": file_contents_local_transform,
         }
 
-    def _send_config_to_kafka(self,
-                              record_set_id,
-                              topic: str,
-                              config_type: str,
-                              content: str) -> Dict[str, Union[str, list]]:
-        key = f'{record_set_id}#{config_type}'
+    def _send_config_to_kafka(
+        self, record_set_id, topic: str, config_type: str, content: str
+    ) -> Dict[str, Union[str, list]]:
+        key = f"{record_set_id}#{config_type}"
         try:
             # TODO: This could be improved to actually check if the returned future succeeds.
             # However this was never a problem so far.
             self.producer.send(topic, key=key, value=content)
-            return {
-                'status': 'SUCCESS'
-            }
+            return {"status": "SUCCESS"}
         except KafkaTimeoutError as err:
             return {
-                'status': 'FATAL',
-                'type': 'KafkaTimeOut',
-                'message': f'Kafka Error: {err}.'
+                "status": "FATAL",
+                "type": "KafkaTimeOut",
+                "message": f"Kafka Error: {err}.",
             }

import_api_app/pull-charts.sh

@@ -8,7 +8,7 @@ TFV_REGISTRY="cr.gitlab.switch.ch/memoriav/memobase-2020/services/import-process
 TFV_CHART_VERSION="1.3.4-chart"
 
 IPD_REGISTRY="cr.gitlab.switch.ch/memoriav/memobase-2020/services/deletion-components/import-process-delete"
-IPD_CHART_VERSION="0.3.1-chart"
+IPD_CHART_VERSION="0.3.8-chart"
 
 helm chart pull ${TFV_REGISTRY}:${TFV_CHART_VERSION}
 helm chart export ${TFV_REGISTRY}:${TFV_CHART_VERSION} -d charts/

import_api_app/resources/clear_cache.py

@@ -21,16 +21,13 @@ from import_api_app.app import app
 
 
 class ClearCache(MethodView):
-
     def __init__(self):
         self.logger = app.logger
-        self.headers = {
-            'X-API-Key': app.config['drupal-api-key']
-        }
-        user = app.config['drupal-user']
-        password = app.config['drupal-password']
+        self.headers = {"X-API-Key": app.config["drupal-api-key"]}
+        user = app.config["drupal-user"]
+        password = app.config["drupal-password"]
         self.auth = HTTPBasicAuth(user, password)
-        self.url = app.config['drupal-api-url'] + app.config['clear-cache-url']
+        self.url = app.config["drupal-api-url"] + app.config["clear-cache-url"]
 
     def get(self):
         """
@@ -48,20 +45,24 @@ class ClearCache(MethodView):
         try:
             response = requests.get(self.url, headers=self.headers, auth=self.auth)
         except Exception as ex:
-            msg = 'Exception while calling ' + self.url + ': ' + str(ex)
+            msg = "Exception while calling " + self.url + ": " + str(ex)
             self.logger.error(msg)
             return {
-                       'response': msg,
+                "response": msg,
             }, 500
         if response.ok:
-            self.logger.debug('successfully called ' + self.url)
+            self.logger.debug("successfully called " + self.url)
             return {
-                       'content': response.content.decode("utf-8"),
+                "content": response.content.decode("utf-8"),
             }, response.status_code
         else:
-            self.logger.error('Clearing Cache Failed')
-            self.logger.error('statuscode: ' + str(response.status_code) + ' content: ' +
-                              response.content.decode('utf-8'))
+            self.logger.error("Clearing Cache Failed")
+            self.logger.error(
+                "statuscode: "
+                + str(response.status_code)
+                + " content: "
+                + response.content.decode("utf-8")
+            )
             return {
-                       'content': response.content.decode("utf-8"),
+                "content": response.content.decode("utf-8"),
             }, response.status_code

import_api_app/resources/delete_institution.py

@@ -21,9 +21,10 @@ from import_api_app.helpers.delete_service import DeleteService
 
 
 class DeleteInstitution(MethodView):
-
     def __init__(self):
-        self.delete_service = DeleteService(app.logger, app.config['NAMESPACE'], app.root_path)
+        self.delete_service = DeleteService(
+            app.logger, app.config["NAMESPACE"], app.root_path
+        )
 
     def post(self, session_id, dryrun=False):
         """
@@ -65,5 +66,7 @@ class DeleteInstitution(MethodView):
                   enum: ['SUCCESS', 'FAILURE']
 
         """
-        institution_id = request.json['institution_id']
-        return self.delete_service.delete_institution(institution_id, session_id, dryrun)
+        institution_id = request.json["institution_id"]
+        return self.delete_service.delete_institution(
+            institution_id, session_id, dryrun
+        )

import_api_app/resources/delete_record.py

@@ -21,9 +21,10 @@ from import_api_app.helpers.delete_service import DeleteService
 
 
 class DeleteRecord(MethodView):
-
     def __init__(self):
-        self.delete_service = DeleteService(app.logger, app.config['NAMESPACE'], app.root_path)
+        self.delete_service = DeleteService(
+            app.logger, app.config["NAMESPACE"], app.root_path
+        )
 
     def post(self, session_id, dryrun=False):
         """
@@ -65,5 +66,5 @@ class DeleteRecord(MethodView):
                   enum: ['SUCCESS', 'FAILURE']
 
         """
-        record_id = request.json['record_id']
+        record_id = request.json["record_id"]
         return self.delete_service.delete_record(record_id, session_id, dryrun)