Commit 715c3502 authored by Jonas Waeber's avatar Jonas Waeber
Browse files

Replace current_app from flask_restful with app from app.py

parent c72da551
Pipeline #34059 passed with stages
in 2 minutes and 11 seconds
......@@ -13,7 +13,8 @@
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
from flask_restful import Resource, current_app
from flask_restful import Resource
from import_api_app.app import app
import requests
import traceback
......@@ -23,14 +24,14 @@ from requests.auth import HTTPBasicAuth
class ClearCache(Resource):
def __init__(self):
self.logger = current_app.logger
self.logger = app.logger
self.headers = {
'X-API-Key': current_app.config['drupal-api-key']
'X-API-Key': app.config['drupal-api-key']
}
user = current_app.config['drupal-user']
password = current_app.config['drupal-password']
user = app.config['drupal-user']
password = app.config['drupal-password']
self.auth = HTTPBasicAuth(user, password)
self.url = current_app.config['drupal-api-url'] + current_app.config['clear-cache-url']
self.url = app.config['drupal-api-url'] + app.config['clear-cache-url']
def get(self):
"""
......
......@@ -14,7 +14,7 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import os
from flask import current_app
from import_api_app.app import app
from flask_restful import Resource
from import_api_app.helm import Helm
from import_api_app.helpers.error import ImportApiError
......@@ -32,7 +32,7 @@ class DeleteService(Resource):
self.do_helm_install(set_values, dryrun)
except Exception as ex:
message = str(ex)
current_app.logger.error(message)
app.logger.error(message)
raise ImportApiError(message)
return -1
return 0
......@@ -47,7 +47,7 @@ class DeleteService(Resource):
self.do_helm_install(set_values, dryrun)
except Exception as ex:
message = str(ex)
current_app.logger.error(message)
app.logger.error(message)
raise ImportApiError(message)
return -1
return 0
......@@ -62,31 +62,31 @@ class DeleteService(Resource):
self.do_helm_install(set_values, dryrun)
except Exception as ex:
message = str(ex)
current_app.logger.error(message)
app.logger.error(message)
raise ImportApiError(message)
return -1
return 0
@staticmethod
def do_helm_install(set_values, dryrun):
current_app.logger.debug(
app.logger.debug(
'calling delete service: type=' +
set_values['deleteObject'] +
' / id=' + set_values['deleteId'] +
' / session=' + set_values['sessionId'])
jobArgs = [
'--'+set_values['deleteObject']+'-filter',
set_values['deleteId'],
set_values['sessionId']
'--' + set_values['deleteObject'] + '-filter',
set_values['deleteId'],
set_values['sessionId']
]
if dryrun:
jobArgs.append('--dry-run')
jobArgs = {'jobArgs': jobArgs}
helm = Helm()
response = helm.install( # noqa: F841
chart=os.path.join(current_app.root_path, "charts", 'dd-marker-prod'),
name=set_values['sessionId']+'-deleter',
namespace=current_app.config['NAMESPACE'],
chart=os.path.join(app.root_path, "charts", 'dd-marker-prod'),
name=set_values['sessionId'] + '-deleter',
namespace=app.config['NAMESPACE'],
set_values=jobArgs,
fail_on_err=False
)
......@@ -13,7 +13,8 @@
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
from flask_restful import Resource, current_app
from flask_restful import Resource
from import_api_app.app import app
from import_api_app.helpers.mapping import MappingFileHandler
......@@ -21,13 +22,13 @@ from import_api_app.helpers.mapping import MappingFileHandler
class FetchMappingFile(Resource):
def __init__(self):
self.host = current_app.config["sftp_host"]
self.port = current_app.config["sftp_port"]
self.user = current_app.config["sftp_user"]
self.password = current_app.config["sftp_password"]
self.base_path = current_app.config['sftp_base_path']
self.topic = current_app.config['topic-configs']
self.kafka_broker_url = current_app.config['kafka-broker-url']
self.host = app.config["sftp_host"]
self.port = app.config["sftp_port"]
self.user = app.config["sftp_user"]
self.password = app.config["sftp_password"]
self.base_path = app.config['sftp_base_path']
self.topic = app.config['topic-configs']
self.kafka_broker_url = app.config['kafka-broker-url']
self.mapping_file_handler = MappingFileHandler(
self.host,
......@@ -37,7 +38,7 @@ class FetchMappingFile(Resource):
self.base_path,
self.kafka_broker_url,
self.topic,
current_app.logger
app.logger
)
def get(self, recordset_id, session_id):
......
......@@ -17,7 +17,8 @@ import hashlib
import os
from flasgger import swag_from
from flask_restful import Resource, reqparse, current_app
from flask_restful import Resource, reqparse
from import_api_app.app import app
from import_api_app.helm import Helm
from import_api_app.helpers.mapping import MappingFileHandler
......@@ -26,13 +27,13 @@ from import_api_app.helpers.mapping import MappingFileHandler
class ImportProcessStart(Resource):
def __init__(self):
self.host = current_app.config["sftp_host"]
self.port = current_app.config["sftp_port"]
self.user = current_app.config["sftp_user"]
self.password = current_app.config["sftp_password"]
self.base_path = current_app.config['sftp_base_path']
self.topic = current_app.config['topic-configs']
self.kafka_broker_url = current_app.config['kafka-broker-url']
self.host = app.config["sftp_host"]
self.port = app.config["sftp_port"]
self.user = app.config["sftp_user"]
self.password = app.config["sftp_password"]
self.base_path = app.config['sftp_base_path']
self.topic = app.config['topic-configs']
self.kafka_broker_url = app.config['kafka-broker-url']
self.mapping_file_handler = MappingFileHandler(
self.host,
......@@ -42,7 +43,7 @@ class ImportProcessStart(Resource):
self.base_path,
self.kafka_broker_url,
self.topic,
current_app.logger
app.logger
)
@swag_from('ImportProcessStart.yml')
......@@ -84,11 +85,11 @@ class ImportProcessStart(Resource):
'sessionId': job_parameters['sessionId'], 'shortSessionId': short_session_id,
'institutionId': job_parameters['institutionId'],
'isPublished': job_parameters['isPublished'],
'kafkaConfigs': current_app.config['tfv-kafka-configs'],
'sftpConfigs': current_app.config['tfv-sftp-configs'],
'topicName': current_app.config['tfv-topic-name'],
'reportingTopicName': current_app.config['tfv-reporting-topic-name'],
'k8sEnvironment': current_app.config['env']
'kafkaConfigs': app.config['tfv-kafka-configs'],
'sftpConfigs': app.config['tfv-sftp-configs'],
'topicName': app.config['tfv-topic-name'],
'reportingTopicName': app.config['tfv-reporting-topic-name'],
'k8sEnvironment': app.config['env']
}
if 'xmlRecordTag' in job_parameters:
......@@ -106,9 +107,9 @@ class ImportProcessStart(Resource):
helm = Helm()
response = helm.install(
chart=os.path.join(current_app.root_path, "charts", 'text-file-validation'),
chart=os.path.join(app.root_path, "charts", 'text-file-validation'),
name=short_session_id + '-' + record_set_id + '-validation',
namespace=current_app.config['NAMESPACE'],
namespace=app.config['NAMESPACE'],
set_values=body,
fail_on_err=False
)
......
......@@ -15,7 +15,8 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import uuid
from flask_restful import Resource, current_app
from flask_restful import Resource
from import_api_app.app import app
from kafka import KafkaProducer
import requests
import json
......@@ -26,7 +27,7 @@ from requests.auth import HTTPBasicAuth
class UpdateInstitution(Resource):
def __init__(self):
self.producer = KafkaProducer(bootstrap_servers=current_app.config['kafka-broker-url'],
self.producer = KafkaProducer(bootstrap_servers=app.config['kafka-broker-url'],
value_serializer=lambda m: json.dumps(m, ensure_ascii=False)
.encode('utf-8'))
......@@ -59,11 +60,11 @@ class UpdateInstitution(Resource):
"""
result = ''
headers = {'X-API-Key': current_app.config['drupal-api-key']}
user = current_app.config['drupal-user']
password = current_app.config['drupal-password']
headers = {'X-API-Key': app.config['drupal-api-key']}
user = app.config['drupal-user']
password = app.config['drupal-password']
auth = HTTPBasicAuth(user, password)
base_url = current_app.config['drupal-api-url']
base_url = app.config['drupal-api-url']
institution_path = f'/jsonapi/node/institution/{institution_drupal_uuid}'
extended_address_path = '/jsonapi/paragraph/extended_address/'
record_set_path = '/jsonapi/node/record_set'
......@@ -115,7 +116,7 @@ class UpdateInstitution(Resource):
)
except LookupError as ex:
msg = f'Could not find key ({institution_drupal_uuid}): {ex}.'
current_app.logger.error(msg)
app.logger.error(msg)
return {
'status': 'FAILURE',
'message': str(ex),
......@@ -123,7 +124,7 @@ class UpdateInstitution(Resource):
}, 500
except Exception as ex:
msg = f'Unknown Exception ({institution_drupal_uuid}): {ex}\n{traceback.format_exc()}'
current_app.logger.error(msg)
app.logger.error(msg)
return {
'status': 'FAILURE',
'message': str(ex),
......@@ -155,7 +156,7 @@ class UpdateInstitution(Resource):
'computed_teaser_color': institution_data_de['attributes']['computed_teaser_color'],
}
try:
producer_topic = current_app.config['topic-drupal-export']
producer_topic = app.config['topic-drupal-export']
headers = [
('recordSetId', bytes('none', encoding='utf-8')),
('sessionId', bytes(str(uuid.uuid4()), encoding='utf-8')),
......@@ -165,16 +166,16 @@ class UpdateInstitution(Resource):
key = bytes(result.get('field_memobase_id'), encoding='utf-8')
self.producer.send(producer_topic, result, key, headers=headers)
except Exception as ex:
msg = f'Unknown Exception ({institution_drupal_uuid}): {ex}. '\
msg = f'Unknown Exception ({institution_drupal_uuid}): {ex}. ' \
f'Check logs for more details.'
current_app.logger.error(f'{msg}\n{traceback.format_exc()}')
app.logger.error(f'{msg}\n{traceback.format_exc()}')
return {
'status': 'FAILURE',
'topic_key': result.get('field_memobase_id'),
'topic_value': result
}, 500
current_app.logger.debug('success for ' + institution_drupal_uuid)
app.logger.debug('success for ' + institution_drupal_uuid)
return {
'status': 'SUCCESS',
'topic_key': result.get('field_memobase_id'),
......
......@@ -15,7 +15,8 @@
# along with this program. If not, see <https://www.gnu.org/licenses/>.
import uuid
from flask_restful import Resource, current_app
from flask_restful import Resource
from import_api_app.app import app
from kafka import KafkaProducer
import requests
import json
......@@ -28,16 +29,16 @@ from requests.auth import HTTPBasicAuth
class UpdateRecordSet(Resource):
def __init__(self):
self.producer = KafkaProducer(bootstrap_servers=current_app.config['kafka-broker-url'],
self.producer = KafkaProducer(bootstrap_servers=app.config['kafka-broker-url'],
value_serializer=lambda m: json.dumps(m, ensure_ascii=False)
.encode('utf-8'))
self.base_url = current_app.config['drupal-api-url']
self.base_url = app.config['drupal-api-url']
self.json_api_path = '/jsonapi/node/record_set/'
self.institution_path = '/jsonapi/node/institution/'
self.metadata_language_path = '/jsonapi/taxonomy_term/language_of_metadata/'
self.headers = {'X-API-Key': current_app.config['drupal-api-key']}
user = current_app.config['drupal-user']
password = current_app.config['drupal-password']
self.headers = {'X-API-Key': app.config['drupal-api-key']}
user = app.config['drupal-user']
password = app.config['drupal-password']
self.auth = HTTPBasicAuth(user, password)
def get(self, record_set_drupal_uuid):
......@@ -216,7 +217,7 @@ class UpdateRecordSet(Resource):
msg = 'LookupError for ' + record_set_drupal_uuid + ': ' + str(ex) + '\n' + \
traceback.format_exc() + '\n' + \
'baseRequest: ' + de_drupal_url + '\n'
current_app.logger.error(msg)
app.logger.error(msg)
return {
'status': 'FAILURE',
'topic_key': result_topic_value.get('field_memobase_id'),
......@@ -226,7 +227,7 @@ class UpdateRecordSet(Resource):
msg = 'Exception for ' + record_set_drupal_uuid + ': ' + str(ex) + '\n' + \
traceback.format_exc() + '\n' + \
'baseRequest: ' + de_drupal_url + '\n'
current_app.logger.error(msg)
app.logger.error(msg)
return {
'status': 'FAILURE',
'topic_key': result_topic_value.get('field_memobase_id'),
......@@ -244,14 +245,14 @@ class UpdateRecordSet(Resource):
]
try:
key = bytes(result_topic_value.get('field_memobase_id'), encoding='utf-8')
current_app.logger.debug(
app.logger.debug(
f'Send message: key={key}, headers={headers}, '
f'message: {json.dumps(result_topic_value, ensure_ascii=False)}')
self.producer.send(current_app.config['topic-drupal-export'], result_topic_value,
self.producer.send(app.config['topic-drupal-export'], result_topic_value,
key=key, headers=headers)
except KafkaTimeoutError as ex:
msg = f'KafkaTimeoutError ({record_set_drupal_id}): {ex}.'
current_app.logger.error(msg)
app.logger.error(msg)
return {
'status': 'FAILURE',
'topic_key': result_topic_value.get('field_memobase_id'),
......@@ -261,7 +262,7 @@ class UpdateRecordSet(Resource):
except Exception as ex:
msg = f'Could not import {result_topic_value.get("field_memobase_id")} ' \
f'(Drupal UUID: {record_set_drupal_id}) (Unknown Exception): ' + str(ex)
current_app.logger.error(f"{msg}\n{traceback.format_exc()}")
app.logger.error(f"{msg}\n{traceback.format_exc()}")
return {
'status': 'FAILURE',
'topic_key': result_topic_value.get('field_memobase_id'),
......@@ -269,7 +270,7 @@ class UpdateRecordSet(Resource):
'exception': msg
}, 503
current_app.logger.debug('success for ' + record_set_drupal_id)
app.logger.debug('success for ' + record_set_drupal_id)
return {
'status': 'SUCCESS',
'topic_key': result_topic_value.get('field_memobase_id'),
......
......@@ -18,7 +18,7 @@ from datetime import datetime
import requests
from flask_restful import Resource, request
from flask_restful import current_app
from import_api_app.app import app
from import_api_app.helpers.error import ImportApiError
from requests.auth import HTTPBasicAuth
......@@ -30,13 +30,13 @@ class WriteJobResultToDrupal(Resource):
self.headers = {
'Content-Type': 'application/vnd.api+json',
'Accept': 'application/vnd.api+json',
'X-API-Key': current_app.config['drupal-api-key']
'X-API-Key': app.config['drupal-api-key']
}
user = current_app.config['drupal-user']
password = current_app.config['drupal-password']
user = app.config['drupal-user']
password = app.config['drupal-password']
self.auth = HTTPBasicAuth(user, password)
self.job_url = f'{current_app.config["drupal-api-url"]}/jsonapi/node/log_result/'
self.import_process_endpoint = f'{current_app.config["drupal-api-url"]}' \
self.job_url = f'{app.config["drupal-api-url"]}/jsonapi/node/log_result/'
self.import_process_endpoint = f'{app.config["drupal-api-url"]}' \
f'/jsonapi/node/import_process/'
def post(self, job_drupal_uuid, job_log_drupal_uuid):
......@@ -114,9 +114,9 @@ class WriteJobResultToDrupal(Resource):
500:
description: There was a problem
"""
self.logger = current_app.logger
self.logger = app.logger
body = request.json
current_app.logger.debug("Report Input Data: \n" + json.dumps(body, indent=2))
app.logger.debug("Report Input Data: \n" + json.dumps(body, indent=2))
try:
fatal = body['fatal']
if fatal == 0:
......@@ -196,26 +196,26 @@ class WriteJobResultToDrupal(Resource):
except requests.exceptions.RequestException:
message = "It was not possible to write to Drupal API \
via the following url " + url
current_app.logger.error(message)
app.logger.error(message)
raise ImportApiError(message)
if response.status_code == 200:
current_app.logger.debug('Updated: ' + url)
app.logger.debug('Updated: ' + url)
result['message'] += 'Updated: ' + url + '\n'
elif response.status_code == 403:
message = "Not authorized to write to: " + url
current_app.logger.error(message)
app.logger.error(message)
raise ImportApiError(message)
elif response.status_code == 404:
message = 'Not Found: ' + url
current_app.logger.error(message)
app.logger.error(message)
raise ImportApiError(message)
elif response.status_code == 500:
message = 'There was an internal server error for ' + url + ': ' + str(response) + \
'. Check the logs for details.'
current_app.logger.error(message)
app.logger.error(message)
raise ImportApiError(message)
else:
message = "Unknown response status code for drupal api for url " + url
current_app.logger.error(message)
app.logger.error(message)
raise ImportApiError(message)
return result
......@@ -16,7 +16,7 @@
import requests
import json
from flask_restful import Resource, request
from flask_restful import current_app
from import_api_app.app import app
from import_api_app.helpers.error import ImportApiError
from requests.auth import HTTPBasicAuth
......@@ -24,14 +24,14 @@ from requests.auth import HTTPBasicAuth
class WriteTypeReportToDrupal(Resource):
def __init__(self):
self.base_url = current_app.config["drupal-api-url"]
self.base_url = app.config["drupal-api-url"]
self.headers = {
'Content-Type': 'application/vnd.api+json',
'Accept': 'application/vnd.api+json',
'X-API-Key': current_app.config['drupal-api-key']
'X-API-Key': app.config['drupal-api-key']
}
user = current_app.config['drupal-user']
password = current_app.config['drupal-password']
user = app.config['drupal-user']
password = app.config['drupal-password']
self.auth = HTTPBasicAuth(user, password)
def post(self):
......@@ -86,7 +86,7 @@ class WriteTypeReportToDrupal(Resource):
node_id, lang_code = self.get_drupal_uuid(identifier, node_type)
return self.write_results(node_type, node_id, lang_code, status, report)
except ImportApiError as e:
current_app.logger.error(
app.logger.error(
f"Could not write report to {node_type}/{identifier}: {e.message}")
return {
'error': f"Could not write report to {node_type}/{identifier}: "
......@@ -101,21 +101,21 @@ class WriteTypeReportToDrupal(Resource):
result = response.json()['data'][0]
return result['id'], result['attributes']['langcode']
except KeyError as er:
current_app.logger.error(f"Could not find the key {er} in data: {response.text}.")
app.logger.error(f"Could not find the key {er} in data: {response.text}.")
raise ImportApiError(f"Could not find the key {er} in data (url={url}): "
f"{response.text}.")
except IndexError:
current_app.logger.error(f"Data field does not contain a list (url={url}): "
f"{response.text}.")
app.logger.error(f"Data field does not contain a list (url={url}): "
f"{response.text}.")
raise ImportApiError(f"Data field does not contain a list (url={url}): "
f"{response.text}.")
else:
current_app.logger.error("Could not find element: " + url)
app.logger.error("Could not find element: " + url)
raise ImportApiError(f"Could not find element for report: {response.text}.")
def write_results(self, node_type, identifier, lang_code, status, report):
current_app.logger.debug(f"writing: {lang_code}/{node_type}/{str(identifier)}"
f"/{str(status)}/{report}")
app.logger.debug(f"writing: {lang_code}/{node_type}/{str(identifier)}"
f"/{str(status)}/{report}")
drupal_type = ''
url = f'{self.base_url}/{lang_code}/jsonapi/node/'
......@@ -151,23 +151,23 @@ class WriteTypeReportToDrupal(Resource):
)
except requests.exceptions.RequestException:
message = f'Request Error: {url}'
current_app.logger.error(message)
app.logger.error(message)
raise ImportApiError(message)
if response.ok:
current_app.logger.debug('Updated: ' + url)
app.logger.debug('Updated: ' + url)
result['message'] += 'Updated: ' + url + '\n'
elif response.status_code == 403:
message = f"Authorization Failed: {url}"
current_app.logger.error(message)
app.logger.error(message)
raise ImportApiError(message)
elif response.status_code == 404:
message = 'Not Found: ' + url
current_app.logger.error(message)
app.logger.error(message)
raise ImportApiError(message)
else:
message = "Unknown response status code for drupal api for url " + url
current_app.logger.error(json.dumps(data, ensure_ascii=False))
current_app.logger.error(message)
current_app.logger.error(response.text)
app.logger.error(json.dumps(data, ensure_ascii=False))
app.logger.error(message)
app.logger.error(response.text)
raise ImportApiError(message)
return result
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment