Commit ed2dbe72 authored by Matthias's avatar Matthias
Browse files

remove unused code from old job-based workflow

parent 338f5335
Pipeline #15100 passed with stages
in 5 minutes and 56 seconds
......@@ -6,9 +6,7 @@ from import_api_app.resources.HelmStart import start
from import_api_app.resources.JobReport import get_report
from import_api_app.resources.WriteJobResultsToDrupal import write_results
from import_api_app.helpers.Drupal import update_status
from kubernetes import client
from threading import Thread
import paramiko
import time
......@@ -24,7 +22,6 @@ class JobStart(Resource):
job_parameters = args['job-parameters']
job_parameters['institutionId'] = institution_id
job_parameters['recordSetId'] = record_set_id
job_parameters['processId'] = process_id
# to debug parameters sent by drupal
current_app.logger.debug(job_parameters)
......@@ -44,49 +41,6 @@ class JobStart(Resource):
process_reporting_topic = f"{process_id}-reporting"
topics.append(process_reporting_topic)
if job_name == 'mapper-service':
# the config map name should be the same as the one defined in the job
# https://gitlab.switch.ch/memoriav/memobase-2020/services/import-process/mapper-service/-/blob/master/helm-charts/templates/job.yaml
config_map_name = process_id + "-mapper-service-mapping-config"
try:
create_config_map_from_file(
record_set_id,
job_parameters['configFileName'],
config_map_name,
current_app.config['NAMESPACE']
)
except ImportApiError as e:
return {'error': e.message}, 500
if job_name == 'normalization-service':
# load mapping file from url
config_map_name = process_id+"-local-normalization-transform"
if job_parameters['hasLocalTransformConfig'] is True:
try:
create_config_map_from_file(
record_set_id,
job_parameters['localTransformFileName'],
config_map_name,
current_app.config['NAMESPACE']
)
except ImportApiError as e:
return {'error': e.message}, 500
if job_name == 'xml-data-transform':
# load mapping file from url
config_map_name = process_id+"-xslt-transform-config"
try:
create_config_map_from_file(
record_set_id,
job_parameters['xsltFileName'],
config_map_name,
current_app.config['NAMESPACE']
)
except ImportApiError as e:
return {'error': e.message}, 500
try:
create_topics(topics)
except ImportApiError as e:
......@@ -138,93 +92,3 @@ def reporting(job_name, job_reporting_topic, drupal_job_log_uuid, drupal_job_uui
update_status(drupal_job_uuid, job_name)
except ImportApiError as e:
return {'error': e.message}, 500
def create_config_map_from_file(record_set_id, config_file_name, config_map_name, namespace):
"""Takes a configuration file (for example mapping.yml) from the SFTP server
in the folder corresponding to the record_set_id and creates a ConfigMap in Kubernetes
using the name specified
Parameters
----------
record_set_id : str
The id of the record_set. This corresponds to the name of the folder on the SFTP server.
The configuration files must be located in the config subfolder
config_file_name : str
The name of the config_file in the sftp folder, for example mapping.yml
Must be a yml file
config_map_name : str
The name of the created ConfigMap in Kubernetes
namespace : str
Kubernetes namespace
"""
try:
config_file_content = fetch_config_file(record_set_id, config_file_name)
# Instantiate the configmap object
metadata = client.V1ObjectMeta(
labels=dict(app="import-api-app"),
name=config_map_name,
namespace=namespace,
)
config_map_dictionary = dict()
config_map_dictionary[config_file_name] = config_file_content
configmap = client.V1ConfigMap(
api_version="v1",
kind="ConfigMap",
data=config_map_dictionary,
metadata=metadata
)
v1 = client.CoreV1Api()
v1.create_namespaced_config_map(
namespace=namespace,
body=configmap
)
except Exception as e:
message = 'impossible to load the local config file file as a config map. ' \
+ record_set_id + '/config/' + config_file_name + str(e)
raise ImportApiError(message)
def fetch_config_file(recordset_id, file_name):
host = current_app.config["sftp_host"]
port = current_app.config["sftp_port"]
user = current_app.config["sftp_user"]
pwd = current_app.config["sftp_password"]
# host = 'mb-wf2.memobase.unibas.ch'
# port = 80
# user = 'mb_sftp'
# pwd = '***'
remote_path = '/swissbib_index/mb_sftp/' + recordset_id + '/config/' + file_name
mapping_file_contents = ''
###
try:
ssh_client = paramiko.SSHClient()
ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh_client.connect(hostname=host,
port=port,
username=user,
password=pwd)
sftp_client = ssh_client.open_sftp()
try:
sftp_client.stat(remote_path)
remote_file = sftp_client.file(remote_path, 'r')
mapping_file_contents = remote_file.read()
mapping_file_contents = mapping_file_contents.decode('utf8')
except IOError:
message = 'remote path "' + remote_path + '" does not exist'
current_app.logger.debug(message)
raise ImportApiError(message)
sftp_client.close()
ssh_client.close()
except Exception as ex:
message = str(ex)
current_app.logger.debug(message)
raise ImportApiError(message)
return mapping_file_contents
......@@ -3,7 +3,6 @@ import import_api_app.helm as helm
from flasgger import swag_from
from import_api_app.utility import generate_helm_name
from subprocess import CalledProcessError
from kubernetes import client
class JobStop(Resource):
......@@ -12,51 +11,12 @@ class JobStop(Resource):
@swag_from('JobStop.yml')
def delete(self, process_id, job_name):
helm_client = helm.Helm()
helm_name = generate_helm_name(process_id, job_name)
# to debug parameters sent by drupal
current_app.logger.debug("process_id : " + process_id)
current_app.logger.debug("job_name : " + job_name)
# todo remove other configmap
if job_name == 'mapper-service':
# delete config map
try:
v1 = client.CoreV1Api()
v1.delete_namespaced_config_map(
# todo : put this suffix in configuration
name=process_id+"-mapper-service-mapping-config",
namespace=current_app.config['NAMESPACE'],
)
except Exception:
return {'error': 'impossible to delete the configmap needed for mapping'}, 500
if job_name == 'normalization-service':
# configmap is optional, we remove it even if is is not there
try:
v1 = client.CoreV1Api()
v1.delete_namespaced_config_map(
name=process_id+"-local-normalization-transform",
namespace=current_app.config['NAMESPACE'],
)
except Exception:
current_app.logger.debug("impossible to delete the "
"normalization configmap, but this can be normal")
if job_name == 'xml-data-transform':
try:
v1 = client.CoreV1Api()
v1.delete_namespaced_config_map(
name=process_id+"-xslt-transform-config",
namespace=current_app.config['NAMESPACE'],
)
except Exception:
return {'error': 'impossible to delete the configmap needed for normalization'}, 500
try:
output = helm_client.uninstall(
name=helm_name,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment