Commit 7913df70 authored by Jonas Waeber's avatar Jonas Waeber
Browse files

Refactor record set endpoint.

parent 47824aa7
Pipeline #26073 passed with stages
in 2 minutes and 38 seconds
......@@ -11,7 +11,7 @@ from import_api_app.resources.WriteJobResultsToDrupal import WriteJobResultToDru
from import_api_app.resources.WriteTypeReportToDrupal import WriteTypeReportToDrupal
# from import_api_app.resources.ReadJobOptionsFromDrupal import ReadJobOptionsFromDrupal
from import_api_app.resources.MonitorDrupalJsonApiInstitution import MonitorDrupalJsonApiInstitution
from import_api_app.resources.MonitorDrupalJsonApiRecordSet import MonitorDrupalJsonApiRecordSet
from import_api_app.resources.UpdateRecordSet import UpdateRecordSet
from import_api_app.resources.ImportProcessStart import ImportProcessStart
from import_api_app.resources.FetchMappingFile import FetchMappingFile
from import_api_app.resources.DeleteRecord import DeleteRecord
......@@ -100,7 +100,7 @@ def create_app(test_config=None):
api.add_resource(WriteJobResultToDrupal, '/v1/drupal/<job_drupal_uuid>/<job_log_drupal_uuid>')
api.add_resource(WriteTypeReportToDrupal, '/v1/drupal/WriteElementReport')
api.add_resource(MonitorDrupalJsonApiInstitution, '/v1/drupal/institution/<institutionId>')
api.add_resource(MonitorDrupalJsonApiRecordSet, '/v1/drupal/recordset/<recordSetId>')
api.add_resource(UpdateRecordSet, '/v1/drupal/recordset/<record_set_id>')
api.add_resource(
DeleteRecord,
'/v1/drupal/delete/record/<session_id>',
......
......@@ -6,19 +6,19 @@ import traceback
from requests.auth import HTTPBasicAuth
class MonitorDrupalJsonApiRecordSet(Resource):
class UpdateRecordSet(Resource):
# Todo write/correct comment for swagger
def get(self, recordSetId):
def get(self, record_set_id):
"""
Get uuid of updated recordset form drupal, gets more information form \
drupal and puts it into the dedicated kafka topic
---
tags:
- Monitor Drupal JSON API
- Import Record Set
parameters:
- in: path
name: recordSetId
name: record_set_id
required: true
description: The UUID of the updated recordset
example: 0c4c777c-94f8-45ba-945a-bfe6967d40da
......@@ -39,170 +39,181 @@ class MonitorDrupalJsonApiRecordSet(Resource):
"""
topicValue = ''
topic_value = ''
# 1. request info from drupal with received uuid
baseRequest = current_app.config['drupal-api-url'] + '/jsonapi/node/' + \
'record_set/' + recordSetId
baseResponse = None
default_request = current_app.config['drupal-api-url'] + '/jsonapi/node/' + \
'record_set/' + record_set_id
base_response = None
user = current_app.config['drupal-user']
password = current_app.config['drupal-password']
auth = HTTPBasicAuth(user, password)
headers = {'X-API-Key': current_app.config['drupal-api-key']}
try:
baseResponse = requests.get(baseRequest, headers=headers, auth=auth)
drupalRecordSet = baseResponse.json()['data']
drupalRecordSet_fr =\
base_response = requests.get(default_request, headers=headers, auth=auth)
drupal_record_set = base_response.json()['data']
drupal_record_set_fr = \
requests.get(current_app.config['drupal-api-url'] + '/fr/jsonapi/node/' +
'record_set/' + recordSetId, headers=headers, auth=auth).json()['data']
drupalRecordSet_it =\
'record_set/' + record_set_id,
headers=headers, auth=auth).json()['data']
drupal_record_set_it = \
requests.get(current_app.config['drupal-api-url'] + '/it/jsonapi/node/' +
'record_set/' + recordSetId, headers=headers, auth=auth).json()['data']
'record_set/' + record_set_id,
headers=headers, auth=auth).json()['data']
institutions = drupalRecordSet['relationships']['field_institution']['data']
institutionIds = self.getInstitutionIdList(institutions, headers, auth)
institutions = drupal_record_set['relationships']['field_institution']['data']
institution_ids = UpdateRecordSet.get_institution_id_list(institutions,
headers, auth)
institutions = drupalRecordSet['relationships']['field_resp_institution_access']['data']
field_resp_institution_access = self.getInstitutionIdList(institutions, headers, auth)
institutions = drupal_record_set['relationships']['field_resp_institution_access'][
'data']
field_resp_institution_access = UpdateRecordSet.get_institution_id_list(
institutions, headers, auth)
institutions = drupalRecordSet['relationships']['field_resp_institution_master']['data']
field_resp_institution_master = self.getInstitutionIdList(institutions, headers, auth)
institutions = drupal_record_set['relationships']['field_resp_institution_master'][
'data']
field_resp_institution_master = UpdateRecordSet.get_institution_id_list(
institutions, headers, auth)
institutions =\
drupalRecordSet['relationships']['field_resp_institution_original']['data']
field_resp_institution_original = self.getInstitutionIdList(institutions, headers, auth)
institutions = \
drupal_record_set['relationships']['field_resp_institution_original']['data']
field_resp_institution_original = UpdateRecordSet.get_institution_id_list(
institutions, headers, auth)
metadataLanguageCodes = []
metadataLanguages =\
drupalRecordSet['relationships']['field_metadata_languages']['data']
for metadataLanguage in metadataLanguages:
drupalRecordSetLanguageCode = \
metadata_language_codes = []
metadata_languages = \
drupal_record_set['relationships']['field_metadata_languages']['data']
for metadataLanguage in metadata_languages:
drupal_record_set_language_code = \
requests.get(current_app.config['drupal-api-url'] + '/jsonapi/taxonomy_term/' +
'language_of_metadata/' + metadataLanguage['id'],
headers=headers, auth=auth)
metadataLanguageCodes.append(
drupalRecordSetLanguageCode.json()['data']['attributes']['name']
metadata_language_codes.append(
drupal_record_set_language_code.json()['data']['attributes']['name']
)
field_related_record_sets = self.getRelatedRecordSets(drupalRecordSet)
field_related_record_sets_fr = self.getRelatedRecordSets(drupalRecordSet_fr)
field_related_record_sets_it = self.getRelatedRecordSets(drupalRecordSet_it)
topicValue = {
'type': drupalRecordSet['type'],
'status': drupalRecordSet['attributes']['status'],
'title_de': drupalRecordSet['attributes']['title'],
'title_fr': drupalRecordSet_fr['attributes']['title'],
'title_it': drupalRecordSet_it['attributes']['title'],
'field_institution': institutionIds,
'field_metadata_language_codes': metadataLanguageCodes,
field_related_record_sets = UpdateRecordSet.get_related_record_sets(
drupal_record_set)
field_related_record_sets_fr = UpdateRecordSet.get_related_record_sets(
drupal_record_set_fr)
field_related_record_sets_it = UpdateRecordSet.get_related_record_sets(
drupal_record_set_it)
topic_value = {
'type': drupal_record_set['type'],
'status': drupal_record_set['attributes']['status'],
'title_de': drupal_record_set['attributes']['title'],
'title_fr': drupal_record_set_fr['attributes']['title'],
'title_it': drupal_record_set_it['attributes']['title'],
'field_institution': institution_ids,
'field_metadata_language_codes': metadata_language_codes,
'computed_teaser_image_url':
drupalRecordSet['attributes']['computed_teaser_image_url'],
drupal_record_set['attributes']['computed_teaser_image_url'],
'field_processed_teaser_text_de':
drupalRecordSet['attributes']['field_processed_teaser_text'],
drupal_record_set['attributes']['field_processed_teaser_text'],
'field_processed_teaser_text_fr':
drupalRecordSet_fr['attributes']['field_processed_teaser_text'],
drupal_record_set_fr['attributes']['field_processed_teaser_text'],
'field_processed_teaser_text_it':
drupalRecordSet_it['attributes']['field_processed_teaser_text'],
drupal_record_set_it['attributes']['field_processed_teaser_text'],
'field_old_memobase_id':
drupalRecordSet['attributes']['field_old_memobase_id'],
'field_access_de': drupalRecordSet['attributes']['field_access'],
'field_access_fr': drupalRecordSet_fr['attributes']['field_access'],
'field_access_it': drupalRecordSet_it['attributes']['field_access'],
drupal_record_set['attributes']['field_old_memobase_id'],
'field_access_de': drupal_record_set['attributes']['field_access'],
'field_access_fr': drupal_record_set_fr['attributes']['field_access'],
'field_access_it': drupal_record_set_it['attributes']['field_access'],
'field_access_memobase_de':
drupalRecordSet['attributes']['field_access_memobase'],
drupal_record_set['attributes']['field_access_memobase'],
'field_access_memobase_fr':
drupalRecordSet_fr['attributes']['field_access_memobase'],
drupal_record_set_fr['attributes']['field_access_memobase'],
'field_access_memobase_it':
drupalRecordSet_it['attributes']['field_access_memobase'],
'field_content_de': drupalRecordSet['attributes']['field_content'],
'field_content_fr': drupalRecordSet_fr['attributes']['field_content'],
'field_content_it': drupalRecordSet_it['attributes']['field_content'],
'field_context_de': drupalRecordSet['attributes']['field_context'],
'field_context_fr': drupalRecordSet_fr['attributes']['field_context'],
'field_context_it': drupalRecordSet_it['attributes']['field_context'],
'field_data_transfer_de': drupalRecordSet['attributes']['field_data_transfer'],
'field_data_transfer_fr': drupalRecordSet_fr['attributes']['field_data_transfer'],
'field_data_transfer_it': drupalRecordSet_it['attributes']['field_data_transfer'],
'field_documents_de': drupalRecordSet['attributes']['field_documents'],
'field_documents_fr': drupalRecordSet_fr['attributes']['field_documents'],
'field_documents_it': drupalRecordSet_it['attributes']['field_documents'],
drupal_record_set_it['attributes']['field_access_memobase'],
'field_content_de': drupal_record_set['attributes']['field_content'],
'field_content_fr': drupal_record_set_fr['attributes']['field_content'],
'field_content_it': drupal_record_set_it['attributes']['field_content'],
'field_context_de': drupal_record_set['attributes']['field_context'],
'field_context_fr': drupal_record_set_fr['attributes']['field_context'],
'field_context_it': drupal_record_set_it['attributes']['field_context'],
'field_data_transfer_de': drupal_record_set['attributes']['field_data_transfer'],
'field_data_transfer_fr': drupal_record_set_fr['attributes']['field_data_transfer'],
'field_data_transfer_it': drupal_record_set_it['attributes']['field_data_transfer'],
'field_documents_de': drupal_record_set['attributes']['field_documents'],
'field_documents_fr': drupal_record_set_fr['attributes']['field_documents'],
'field_documents_it': drupal_record_set_it['attributes']['field_documents'],
'field_info_on_development_de':
drupalRecordSet['attributes']['field_info_on_development'],
drupal_record_set['attributes']['field_info_on_development'],
'field_info_on_development_fr':
drupalRecordSet_fr['attributes']['field_info_on_development'],
drupal_record_set_fr['attributes']['field_info_on_development'],
'field_info_on_development_it':
drupalRecordSet_it['attributes']['field_info_on_development'],
'field_language_de': drupalRecordSet['attributes']['field_language'],
'field_language_fr': drupalRecordSet_fr['attributes']['field_language'],
'field_language_it': drupalRecordSet_it['attributes']['field_language'],
'field_memobase_id': drupalRecordSet['attributes']['field_memobase_id'],
'field_notes': drupalRecordSet['attributes']['field_notes'],
drupal_record_set_it['attributes']['field_info_on_development'],
'field_language_de': drupal_record_set['attributes']['field_language'],
'field_language_fr': drupal_record_set_fr['attributes']['field_language'],
'field_language_it': drupal_record_set_it['attributes']['field_language'],
'field_memobase_id': drupal_record_set['attributes']['field_memobase_id'],
'field_notes': drupal_record_set['attributes']['field_notes'],
'field_original_description_de':
drupalRecordSet['attributes']['field_original_description'],
drupal_record_set['attributes']['field_original_description'],
'field_original_description_fr':
drupalRecordSet_fr['attributes']['field_original_description'],
drupal_record_set_fr['attributes']['field_original_description'],
'field_original_description_it':
drupalRecordSet_it['attributes']['field_original_description'],
'field_original_id': drupalRecordSet['attributes']['field_original_id'],
drupal_record_set_it['attributes']['field_original_description'],
'field_original_id': drupal_record_set['attributes']['field_original_id'],
'field_original_shelf_mark':
drupalRecordSet['attributes']['field_original_shelf_mark'],
'field_original_title_de': drupalRecordSet['attributes']['field_original_title'],
drupal_record_set['attributes']['field_original_shelf_mark'],
'field_original_title_de': drupal_record_set['attributes']['field_original_title'],
'field_original_title_fr':
drupalRecordSet_fr['attributes']['field_original_title'],
drupal_record_set_fr['attributes']['field_original_title'],
'field_original_title_it':
drupalRecordSet_it['attributes']['field_original_title'],
'field_project_de': drupalRecordSet['attributes']['field_project'],
'field_project_fr': drupalRecordSet_fr['attributes']['field_project'],
'field_project_it': drupalRecordSet_it['attributes']['field_project'],
'field_publications_de': drupalRecordSet['attributes']['field_publications'],
'field_publications_fr': drupalRecordSet_fr['attributes']['field_publications'],
'field_publications_it': drupalRecordSet_it['attributes']['field_publications'],
drupal_record_set_it['attributes']['field_original_title'],
'field_project_de': drupal_record_set['attributes']['field_project'],
'field_project_fr': drupal_record_set_fr['attributes']['field_project'],
'field_project_it': drupal_record_set_it['attributes']['field_project'],
'field_publications_de': drupal_record_set['attributes']['field_publications'],
'field_publications_fr': drupal_record_set_fr['attributes']['field_publications'],
'field_publications_it': drupal_record_set_it['attributes']['field_publications'],
'field_related_record_sets_de': field_related_record_sets,
'field_related_record_sets_fr': field_related_record_sets_fr,
'field_related_record_sets_it': field_related_record_sets_it,
'field_rights_de': drupalRecordSet['attributes']['field_rights'],
'field_rights_fr': drupalRecordSet_fr['attributes']['field_rights'],
'field_rights_it': drupalRecordSet_it['attributes']['field_rights'],
'field_scope_de': drupalRecordSet['attributes']['field_scope'],
'field_scope_fr': drupalRecordSet_fr['attributes']['field_scope'],
'field_scope_it': drupalRecordSet_it['attributes']['field_scope'],
'field_selection_de': drupalRecordSet['attributes']['field_selection'],
'field_selection_fr': drupalRecordSet_fr['attributes']['field_selection'],
'field_selection_it': drupalRecordSet_it['attributes']['field_selection'],
'field_rights_de': drupal_record_set['attributes']['field_rights'],
'field_rights_fr': drupal_record_set_fr['attributes']['field_rights'],
'field_rights_it': drupal_record_set_it['attributes']['field_rights'],
'field_scope_de': drupal_record_set['attributes']['field_scope'],
'field_scope_fr': drupal_record_set_fr['attributes']['field_scope'],
'field_scope_it': drupal_record_set_it['attributes']['field_scope'],
'field_selection_de': drupal_record_set['attributes']['field_selection'],
'field_selection_fr': drupal_record_set_fr['attributes']['field_selection'],
'field_selection_it': drupal_record_set_it['attributes']['field_selection'],
'field_supported_by_memoriav':
drupalRecordSet['attributes']['field_supported_by_memoriav'],
'field_time_period': drupalRecordSet['attributes']['field_time_period'],
'field_transfer_date': drupalRecordSet['attributes']['field_transfer_date'],
drupal_record_set['attributes']['field_supported_by_memoriav'],
'field_time_period': drupal_record_set['attributes']['field_time_period'],
'field_transfer_date': drupal_record_set['attributes']['field_transfer_date'],
'field_image_gallery':
drupalRecordSet['relationships']['field_image_gallery'],
'field_metadata_languages': metadataLanguageCodes,
drupal_record_set['relationships']['field_image_gallery'],
'field_metadata_languages': metadata_language_codes,
'field_resp_institution_access': field_resp_institution_access,
'field_resp_institution_master': field_resp_institution_master,
'field_resp_institution_original': field_resp_institution_original,
'field_teaser_image': drupalRecordSet['relationships']['field_teaser_image']
'field_teaser_image': drupal_record_set['relationships']['field_teaser_image']
}
except LookupError as ex:
msg = 'LookupError for ' + recordSetId + ': ' + str(ex) + '\n' +\
msg = 'LookupError for ' + record_set_id + ': ' + str(ex) + '\n' + \
traceback.format_exc() + '\n' + \
'baseRequest: ' + baseRequest + '\n' + \
'baseResponse: ' + baseResponse.content.decode("utf-8")
'baseRequest: ' + default_request + '\n' + \
'baseResponse: ' + base_response.content.decode("utf-8")
current_app.logger.error(msg)
return {
'status': 'FAILURE',
'topic_key': topicValue.get('field_memobase_id'),
'topic_value': topicValue
'topic_key': topic_value.get('field_memobase_id'),
'topic_value': topic_value
}, 500
except Exception as ex:
msg = 'Exception for ' + recordSetId + ': ' + str(ex) + '\n' + \
msg = 'Exception for ' + record_set_id + ': ' + str(ex) + '\n' + \
traceback.format_exc() + '\n' + \
'baseRequest: ' + baseRequest + '\n' + \
'baseResponse: ' + baseResponse.content.decode("utf-8")
'baseRequest: ' + default_request + '\n' + \
'baseResponse: ' + base_response.content.decode("utf-8")
current_app.logger.error(msg)
return {
'status': 'FAILURE',
'topic_key': topicValue.get('field_memobase_id'),
'topic_value': topicValue,
'topic_key': topic_value.get('field_memobase_id'),
'topic_value': topic_value,
}, 500
# 2. write info into kafka topic
......@@ -210,44 +221,45 @@ class MonitorDrupalJsonApiRecordSet(Resource):
producer = KafkaProducer(bootstrap_servers=current_app.config['kafka-broker-url'],
value_serializer=lambda m: json.dumps(m, ensure_ascii=False)
.encode('utf-8'))
key = bytes(topicValue.get('field_memobase_id'), encoding='utf-8')
producer.send(current_app.config['topic-drupal-export'], topicValue, key=key)
key = bytes(topic_value.get('field_memobase_id'), encoding='utf-8')
producer.send(current_app.config['topic-drupal-export'], topic_value, key=key)
except Exception as ex:
msg = 'Exception for ' + recordSetId + ': ' + str(ex) + '\n' +\
msg = 'Exception for ' + record_set_id + ': ' + str(ex) + '\n' + \
traceback.format_exc()
current_app.logger.error(msg)
return {
'status': 'FAILURE',
'topic_key': topicValue.get('field_memobase_id'),
'topic_value': topicValue,
'topic_key': topic_value.get('field_memobase_id'),
'topic_value': topic_value,
'exception': msg
}, 500
current_app.logger.debug('success for ' + recordSetId)
current_app.logger.debug('success for ' + record_set_id)
return {
'status': 'SUCCESS',
'topic_key': topicValue.get('field_memobase_id'),
'topic_value': topicValue
'topic_key': topic_value.get('field_memobase_id'),
'topic_value': topic_value
}, 200
def getInstitutionIdList(self, institutionData, headers, auth):
institutionIds = []
for institution in institutionData:
institution
drupalInstitution = \
@staticmethod
def get_institution_id_list(institution_data, headers, auth):
institution_ids = []
for institution in institution_data:
drupal_institution = \
requests.get(current_app.config['drupal-api-url'] + '/jsonapi/node/' +
'institution/' + institution['id'], headers=headers, auth=auth)
institutionIds.append(
drupalInstitution.json()['data']['attributes']['field_memobase_id']
institution_ids.append(
drupal_institution.json()['data']['attributes']['field_memobase_id']
)
return institutionIds
return institution_ids
def getRelatedRecordSets(self, fields):
relatedRecordSets = fields['attributes']['field_related_record_sets']
revisedRelatedRecordSets = []
for relatedRecordSet in relatedRecordSets:
@staticmethod
def get_related_record_sets(fields):
related_record_sets = fields['attributes']['field_related_record_sets']
revised_related_record_sets = []
for relatedRecordSet in related_record_sets:
if 'entity:node' in relatedRecordSet['uri']:
relatedRecordSet['uri'] = fields['attributes']['field_memobase_id']
relatedRecordSet['title'] = 'internal'
revisedRelatedRecordSets.append(relatedRecordSet)
return revisedRelatedRecordSets
revised_related_record_sets.append(relatedRecordSet)
return revised_related_record_sets
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment