Skip to content

GitLab

Commit 40904fc4 authored by Matthias's avatar Matthias
Browse files

add endpoint for clearing drupal cache (+merge previous commits)

parents 896f8562 39dcb978
Pipeline #26130 passed with stages
in 3 minutes and 17 seconds
Show whitespace changes
Inline Side-by-side
import_api_app/__init__.py
View file @ 40904fc4
......@@ -11,8 +11,8 @@ from import_api_app.resources.WriteJobResultsToDrupal import WriteJobResultToDru
from import_api_app.resources.WriteTypeReportToDrupal import WriteTypeReportToDrupal
# from import_api_app.resources.ReadJobOptionsFromDrupal import ReadJobOptionsFromDrupal
from import_api_app.resources.MonitorDrupalJsonApiInstitution import MonitorDrupalJsonApiInstitution
from import_api_app.resources.MonitorDrupalJsonApiRecordSet import MonitorDrupalJsonApiRecordSet
from import_api_app.resources.ClearCache import ClearCache
from import_api_app.resources.UpdateRecordSet import UpdateRecordSet
from import_api_app.resources.ImportProcessStart import ImportProcessStart
from import_api_app.resources.FetchMappingFile import FetchMappingFile
from import_api_app.resources.DeleteRecord import DeleteRecord
......@@ -101,7 +101,7 @@ def create_app(test_config=None):
api.add_resource(WriteJobResultToDrupal, '/v1/drupal/<job_drupal_uuid>/<job_log_drupal_uuid>')
api.add_resource(WriteTypeReportToDrupal, '/v1/drupal/WriteElementReport')
api.add_resource(MonitorDrupalJsonApiInstitution, '/v1/drupal/institution/<institutionId>')
api.add_resource(MonitorDrupalJsonApiRecordSet, '/v1/drupal/recordset/<recordSetId>')
api.add_resource(UpdateRecordSet, '/v1/drupal/recordset/<record_set_id>')
api.add_resource(
DeleteRecord,
'/v1/drupal/delete/record/<session_id>',
......
import_api_app/resources/MonitorDrupalJsonApiInstitution.py
View file @ 40904fc4
......@@ -71,8 +71,10 @@ class MonitorDrupalJsonApiInstitution(Resource):
'extended_address/' + addressElement['id'],
headers=headers, auth=auth).json()['data']
fieldAdressess = drupalResponseAddress['attributes']['field_address']
if fieldAdressess is not None:
fieldAdressess['coordinates'] = \
drupalResponseAddress['attributes']['field_geographical_coordinates']['value']
drupalResponseAddress['attributes']['field_geographical_coordinates'][
'value']
addresses.append(fieldAdressess)
institutionTypes = []
......@@ -102,7 +104,7 @@ class MonitorDrupalJsonApiInstitution(Resource):
current_app.logger.error(msg)
return {
'status': 'FAILURE',
'message': ex,
'message': str(ex),
'topic_value': topic_value
}, 500
except Exception as ex:
......@@ -111,7 +113,7 @@ class MonitorDrupalJsonApiInstitution(Resource):
current_app.logger.error(msg)
return {
'status': 'FAILURE',
'message': ex,
'message': str(ex),
'topic_value': topic_value
}, 500
topic_value = {
......
import_api_app/resources/UpdateRecordSet.py 0 → 100644
View file @ 40904fc4
from flask_restful import Resource, current_app
from kafka import KafkaProducer
import requests
import json
import traceback
from requests.auth import HTTPBasicAuth
class UpdateRecordSet(Resource):
# Todo write/correct comment for swagger
def get(self, record_set_id):
"""
Get uuid of updated recordset form drupal, gets more information form \
drupal and puts it into the dedicated kafka topic
---
tags:
- Import Record Set
parameters:
- in: path
name: record_set_id
required: true
description: The UUID of the updated recordset
example: 0c4c777c-94f8-45ba-945a-bfe6967d40da
type: string
responses:
200:
description: Success, the information has been written into
the kafka topic
schema:
properties:
status:
type: string
example: SUCCESS/FAILURE
enum: ['SUCCESS', 'FAILURE']
result_topic_value:
type: string/json
example: the value written into the topic
"""
result_topic_value = ''
base_url = current_app.config['drupal-api-url']
json_api_path = '/jsonapi/node/record_set/'
metadata_language_path = '/jsonapi/taxonomy_term/language_of_metadata/'
user = current_app.config['drupal-user']
password = current_app.config['drupal-password']
auth = HTTPBasicAuth(user, password)
headers = {'X-API-Key': current_app.config['drupal-api-key']}
# Retrieve Drupal Entities for each language.
# Returns default entity if there is a language missing.
de_drupal_url = f'{base_url}/de{json_api_path}{record_set_id}'
fr_drupal_url = f'{base_url}/fr{json_api_path}{record_set_id}'
it_drupal_url = f'{base_url}/it{json_api_path}{record_set_id}'
try:
drupal_record_set_de = \
requests.get(de_drupal_url, headers=headers, auth=auth).json()['data']
drupal_record_set_fr = \
requests.get(fr_drupal_url, headers=headers, auth=auth).json()['data']
drupal_record_set_it = \
requests.get(it_drupal_url, headers=headers, auth=auth).json()['data']
institutions = drupal_record_set_de['relationships']['field_institution']['data']
institution_ids = UpdateRecordSet.get_institution_id_list(institutions, headers, auth)
institutions = \
drupal_record_set_de['relationships']['field_resp_institution_access']['data']
access_institution_ids = UpdateRecordSet.get_institution_id_list(
institutions, headers, auth)
institutions = \
drupal_record_set_de['relationships']['field_resp_institution_master']['data']
master_institutions_ids = UpdateRecordSet.get_institution_id_list(
institutions, headers, auth)
institutions = \
drupal_record_set_de['relationships']['field_resp_institution_original']['data']
original_institution_ids = UpdateRecordSet.get_institution_id_list(
institutions, headers, auth)
metadata_language_codes = []
metadata_languages = \
drupal_record_set_de['relationships']['field_metadata_languages']['data']
for metadataLanguage in metadata_languages:
drupal_record_set_language_code = \
requests.get(f'{base_url}{metadata_language_path}{metadataLanguage["id"]}',
headers=headers, auth=auth)
metadata_language_codes.append(
drupal_record_set_language_code.json()['data']['attributes']['name']
)
related_record_sets_de = \
UpdateRecordSet.get_related_record_sets(drupal_record_set_de)
related_record_sets_fr = \
UpdateRecordSet.get_related_record_sets(drupal_record_set_fr)
related_record_sets_it = \
UpdateRecordSet.get_related_record_sets(drupal_record_set_it)
result_topic_value = {
'type': drupal_record_set_de['type'],
'status': drupal_record_set_de['attributes']['status'],
'title_de': drupal_record_set_de['attributes']['title'],
'title_fr': drupal_record_set_fr['attributes']['title'],
'title_it': drupal_record_set_it['attributes']['title'],
'field_institution': institution_ids,
'field_metadata_language_codes': metadata_language_codes,
'computed_teaser_image_url':
drupal_record_set_de['attributes']['computed_teaser_image_url'],
'field_processed_teaser_text_de':
drupal_record_set_de['attributes']['field_processed_teaser_text'],
'field_processed_teaser_text_fr':
drupal_record_set_fr['attributes']['field_processed_teaser_text'],
'field_processed_teaser_text_it':
drupal_record_set_it['attributes']['field_processed_teaser_text'],
'field_old_memobase_id':
drupal_record_set_de['attributes']['field_old_memobase_id'],
'field_access_de': drupal_record_set_de['attributes']['field_access'],
'field_access_fr': drupal_record_set_fr['attributes']['field_access'],
'field_access_it': drupal_record_set_it['attributes']['field_access'],
'field_access_memobase_de':
drupal_record_set_de['attributes']['field_access_memobase'],
'field_access_memobase_fr':
drupal_record_set_fr['attributes']['field_access_memobase'],
'field_access_memobase_it':
drupal_record_set_it['attributes']['field_access_memobase'],
'field_content_de': drupal_record_set_de['attributes']['field_content'],
'field_content_fr': drupal_record_set_fr['attributes']['field_content'],
'field_content_it': drupal_record_set_it['attributes']['field_content'],
'field_context_de': drupal_record_set_de['attributes']['field_context'],
'field_context_fr': drupal_record_set_fr['attributes']['field_context'],
'field_context_it': drupal_record_set_it['attributes']['field_context'],
'field_data_transfer_de': drupal_record_set_de['attributes']['field_data_transfer'],
'field_data_transfer_fr': drupal_record_set_fr['attributes']['field_data_transfer'],
'field_data_transfer_it': drupal_record_set_it['attributes']['field_data_transfer'],
'field_documents_de': drupal_record_set_de['attributes']['field_documents'],
'field_documents_fr': drupal_record_set_fr['attributes']['field_documents'],
'field_documents_it': drupal_record_set_it['attributes']['field_documents'],
'field_info_on_development_de':
drupal_record_set_de['attributes']['field_info_on_development'],
'field_info_on_development_fr':
drupal_record_set_fr['attributes']['field_info_on_development'],
'field_info_on_development_it':
drupal_record_set_it['attributes']['field_info_on_development'],
'field_language_de': drupal_record_set_de['attributes']['field_language'],
'field_language_fr': drupal_record_set_fr['attributes']['field_language'],
'field_language_it': drupal_record_set_it['attributes']['field_language'],
'field_memobase_id': drupal_record_set_de['attributes']['field_memobase_id'],
'field_notes': drupal_record_set_de['attributes']['field_notes'],
'field_original_description_de':
drupal_record_set_de['attributes']['field_original_description'],
'field_original_description_fr':
drupal_record_set_fr['attributes']['field_original_description'],
'field_original_description_it':
drupal_record_set_it['attributes']['field_original_description'],
'field_original_id': drupal_record_set_de['attributes']['field_original_id'],
'field_original_shelf_mark':
drupal_record_set_de['attributes']['field_original_shelf_mark'],
'field_original_title_de': drupal_record_set_de['attributes'][
'field_original_title'],
'field_original_title_fr':
drupal_record_set_fr['attributes']['field_original_title'],
'field_original_title_it':
drupal_record_set_it['attributes']['field_original_title'],
'field_project_de': drupal_record_set_de['attributes']['field_project'],
'field_project_fr': drupal_record_set_fr['attributes']['field_project'],
'field_project_it': drupal_record_set_it['attributes']['field_project'],
'field_publications_de': drupal_record_set_de['attributes']['field_publications'],
'field_publications_fr': drupal_record_set_fr['attributes']['field_publications'],
'field_publications_it': drupal_record_set_it['attributes']['field_publications'],
'field_related_record_sets_de': related_record_sets_de,
'field_related_record_sets_fr': related_record_sets_fr,
'field_related_record_sets_it': related_record_sets_it,
'field_rights_de': drupal_record_set_de['attributes']['field_rights'],
'field_rights_fr': drupal_record_set_fr['attributes']['field_rights'],
'field_rights_it': drupal_record_set_it['attributes']['field_rights'],
'field_scope_de': drupal_record_set_de['attributes']['field_scope'],
'field_scope_fr': drupal_record_set_fr['attributes']['field_scope'],
'field_scope_it': drupal_record_set_it['attributes']['field_scope'],
'field_selection_de': drupal_record_set_de['attributes']['field_selection'],
'field_selection_fr': drupal_record_set_fr['attributes']['field_selection'],
'field_selection_it': drupal_record_set_it['attributes']['field_selection'],
'field_supported_by_memoriav':
drupal_record_set_de['attributes']['field_supported_by_memoriav'],
'field_time_period': drupal_record_set_de['attributes']['field_time_period'],
'field_transfer_date': drupal_record_set_de['attributes']['field_transfer_date'],
'field_image_gallery':
drupal_record_set_de['relationships']['field_image_gallery'],
'field_metadata_languages': metadata_language_codes,
'field_resp_institution_access': access_institution_ids,
'field_resp_institution_master': master_institutions_ids,
'field_resp_institution_original': original_institution_ids,
'field_teaser_image': drupal_record_set_de['relationships']['field_teaser_image']
}
except LookupError as ex:
msg = 'LookupError for ' + record_set_id + ': ' + str(ex) + '\n' + \
traceback.format_exc() + '\n' + \
'baseRequest: ' + de_drupal_url + '\n'
current_app.logger.error(msg)
return {
'status': 'FAILURE',
'topic_key': result_topic_value.get('field_memobase_id'),
'result_topic_value': result_topic_value
}, 500
except Exception as ex:
msg = 'Exception for ' + record_set_id + ': ' + str(ex) + '\n' + \
traceback.format_exc() + '\n' + \
'baseRequest: ' + de_drupal_url + '\n'
current_app.logger.error(msg)
return {
'status': 'FAILURE',
'topic_key': result_topic_value.get('field_memobase_id'),
'result_topic_value': result_topic_value,
}, 500
# 2. write info into kafka topic
try:
producer = KafkaProducer(bootstrap_servers=current_app.config['kafka-broker-url'],
value_serializer=lambda m: json.dumps(m, ensure_ascii=False)
.encode('utf-8'))
key = bytes(result_topic_value.get('field_memobase_id'), encoding='utf-8')
producer.send(current_app.config['topic-drupal-export'], result_topic_value, key=key)
except Exception as ex:
msg = 'Exception for ' + record_set_id + ': ' + str(ex) + '\n' + \
traceback.format_exc()
current_app.logger.error(msg)
return {
'status': 'FAILURE',
'topic_key': result_topic_value.get('field_memobase_id'),
'result_topic_value': result_topic_value,
'exception': msg
}, 500
current_app.logger.debug('success for ' + record_set_id)
return {
'status': 'SUCCESS',
'topic_key': result_topic_value.get('field_memobase_id'),
'result_topic_value': result_topic_value
}, 200
@staticmethod
def get_institution_id_list(institution_data, headers, auth):
institution_ids = []
for institution in institution_data:
drupal_institution = \
requests.get(current_app.config['drupal-api-url'] + '/jsonapi/node/' +
'institution/' + institution['id'], headers=headers, auth=auth)
institution_ids.append(
drupal_institution.json()['data']['attributes']['field_memobase_id']
)
return institution_ids
@staticmethod
def get_related_record_sets(fields):
related_record_sets = fields['attributes']['field_related_record_sets']
revised_related_record_sets = []
for relatedRecordSet in related_record_sets:
if 'entity:node' in relatedRecordSet['uri']:
relatedRecordSet['uri'] = fields['attributes']['field_memobase_id']
relatedRecordSet['title'] = 'internal'
revised_related_record_sets.append(relatedRecordSet)
return revised_related_record_sets
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment