Commit 1a7d09e0 authored by Matthias's avatar Matthias
Browse files

implement memo-515, throw exception if envvars not present

parent ed096c68
Pipeline #13418 passed with stages
in 7 minutes
......@@ -17,10 +17,13 @@ from import_api_app.resources.WriteJobResultsToDrupal import WriteJobResultToDru
from import_api_app.resources.ReadJobOptionsFromDrupal import ReadJobOptionsFromDrupal
from import_api_app.resources.MonitorDrupalJsonApiInstitution import MonitorDrupalJsonApiInstitution
from import_api_app.resources.MonitorDrupalJsonApiRecordSet import MonitorDrupalJsonApiRecordSet
from import_api_app.resources.FetchMappingFile import FetchMappingFile
# from import_api_app.resources.HelmStart import HelmStart
# from import_api_app.resources.HelmStop import HelmStop
import os
from models.Error import ImportApiError
def create_app(test_config=None):
app = Flask(__name__)
......@@ -32,23 +35,29 @@ def create_app(test_config=None):
api = Api(app)
# todo : raise an exception when environment variable not present
app.config['kafka-broker-url'] = os.environ['KAFKA_BOOTSTRAP_SERVERS']
app.config['drupal-api-key'] = os.environ['DRUPAL_API_KEY']
app.config['SWAGGER'] = {
'title': 'Memobase Import API',
'version': 'dev',
'uiversion': 3,
'termsOfService': 'http://memobase.ch/de/disclaimer',
'description': 'API to start, stop, manage import processes for '
'memobase. Will be used in the Admin Interface (Drupal).',
'contact': {
'name': 'UB Basel',
'url': 'https://ub.unibas.ch',
'email': 'swissbib-ub@unibas.ch'},
'favicon': '/favicon.ico'}
Swagger(app)
try:
app.config['kafka-broker-url'] = os.environ['KAFKA_BOOTSTRAP_SERVERS']
app.config['drupal-api-key'] = os.environ['DRUPAL_API_KEY']
app.config['sftp_host'] = os.environ['SFTP_HOST']
app.config['sftp_port'] = os.environ['SFTP_PORT']
app.config['sftp_user'] = os.environ['SFTP_USER']
app.config['sftp_password'] = os.environ['SFTP_PASSWORD']
app.config['SWAGGER'] = {
'title': 'Memobase Import API',
'version': 'dev',
'uiversion': 3,
'termsOfService': 'http://memobase.ch/de/disclaimer',
'description': 'API to start, stop, manage import processes for '
'memobase. Will be used in the Admin Interface (Drupal).',
'contact': {
'name': 'UB Basel',
'url': 'https://ub.unibas.ch',
'email': 'swissbib-ub@unibas.ch'},
'favicon': '/favicon.ico'}
Swagger(app)
except Exception as ex:
raise ImportApiError(str(ex))
@app.route("/")
def home():
......@@ -83,6 +92,8 @@ def create_app(test_config=None):
api.add_resource(MonitorDrupalJsonApiInstitution, '/v1/drupal/institution/<institutionId>')
api.add_resource(MonitorDrupalJsonApiRecordSet, '/v1/drupal/recordset/<recordSetId>')
api.add_resource(FetchMappingFile, '/v1/mappingfile/<recordset_id>/<mapping_file_name>')
api.add_resource(KafkaTopics, '/v1/kafka/topics')
api.add_resource(JobReport, '/v1/job/<topic_name>/report')
......
from flask_restful import Resource, current_app
import paramiko
import yaml
class FetchMappingFile(Resource):
def get(self, recordset_id, mapping_file_name):
"""
Fetches the mapping file form the sftp-server
---
tags:
- fetch mapping file
parameters:
- in: path
name: recordset_id
required: true
description: The name of the record set (matches folder name on sftp)
example: AfZ-Becker-Audiovisuals
type: string
- in: path
name: mapping_file_name
required: true
description: The name of the mapping file
example: mapping.yml
type: string
responses:
200:
description: Success, the mapping file has been retrieved
schema:
properties:
status:
type: string
example: SUCCESS/FAILURE
enum: ['SUCCESS', 'FAILURE']
contents:
type: string/yml
example: the contents of the mapping file...
"""
host = current_app.config["sftp_host"]
port = current_app.config["sftp_port"]
user = current_app.config["sftp_user"]
pwd = current_app.config["sftp_password"]
# host = 'mb-wf2.memobase.unibas.ch'
# port = 80
# user = 'mb_sftp'
# pwd = '***'
remotePath = '/swissbib_index/mb_sftp/' + recordset_id + '/config/' + mapping_file_name
mappingFileContents = ''
status = 'SUCCESS'
###
try:
sshClient = paramiko.SSHClient()
sshClient.set_missing_host_key_policy(paramiko.AutoAddPolicy())
sshClient.connect(hostname=host,
port=port,
username=user,
password=pwd)
sftpClient = sshClient.open_sftp()
try:
sftpClient.stat(remotePath)
remoteFile = sftpClient.file(remotePath, 'r')
mappingFileContents = remoteFile.read()
mappingFileContents = mappingFileContents.decode('utf8')
mappingFileContentsJson = yaml.load(mappingFileContents)
returnCode = 200
except IOError:
status = 'remote path "' + remotePath + '" does not exist'
returnCode = 500
sftpClient.close()
sshClient.close()
except Exception as ex:
returnCode = 500
status = str(ex)
current_app.logger.debug(str(ex))
return {'status': status, 'contents': mappingFileContentsJson}, returnCode
......@@ -16,6 +16,8 @@ from kubernetes import client
from threading import Thread
import requests
import yaml
import json
class JobStart(Resource):
......@@ -54,17 +56,13 @@ class JobStart(Resource):
# load mapping file from url
try:
# only allow mapping file at url starting with
mapping_file_url = job_parameters['configFileName']
if not(mapping_file_url.startswith(
'https://gitlab.switch.ch/memoriav/memobase-2020/'
)):
return {
'error': 'The url of the mapping file should start'
'with https://gitlab.switch.ch/memoriav/memobase-2020/'
}, 500
mapping_file_url = 'https://import.memobase.k8s.unibas.ch/v1/mappingfile/' +\
job_parameters['recordSetId'] + '/config/' + 'mapping.yml'
# mapping_file_url = 'http://0.0.0.0:5000/v1/mappingfile/' +\
# job_parameters['recordSetId'] + '/' + 'mapping.yml'
response = requests.get(mapping_file_url)
mapping_file_content = response.text
mapping_file_content = response.content
mapping_file_content = yaml.dump(json.loads(mapping_file_content))
config_map_name = process_id+"-mapper-service-mapping-config"
# Instantiate the configmap object
......@@ -86,15 +84,15 @@ class JobStart(Resource):
job_parameters['configMapName'] = config_map_name
v1 = client.CoreV1Api()
v1.create_namespaced_config_map(
namespace=current_app.config['NAMESPACE'],
body=configmap
)
except Exception:
except Exception as ex:
return {
'error': 'impossible to load the mapping file as a config map. Url: '
+ mapping_file_url
+ mapping_file_url,
'exception': str(ex)
}, 500
try:
......
......@@ -23,6 +23,8 @@ spec:
envFrom:
- secretRef:
name: drupal-api-credentials
- secretRef:
name: internal-sftp-config
- configMapRef:
name: "prod-kafka-bootstrap-servers"
ports:
......
......@@ -9,4 +9,5 @@ pre-commit
jsonapi-requests
requests
kafka-python
pysftp
pyyaml
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment