from flask_restful import Resource, current_app from kafka import KafkaProducer import requests import json import traceback from requests.auth import HTTPBasicAuth class UpdateRecordSet(Resource): # Todo write/correct comment for swagger def get(self, record_set_id): """ Get uuid of updated recordset form drupal, gets more information form \ drupal and puts it into the dedicated kafka topic --- tags: - Import Record Set parameters: - in: path name: record_set_id required: true description: The UUID of the updated recordset example: 0c4c777c-94f8-45ba-945a-bfe6967d40da type: string responses: 200: description: Success, the information has been written into the kafka topic schema: properties: status: type: string example: SUCCESS/FAILURE enum: ['SUCCESS', 'FAILURE'] result_topic_value: type: string/json example: the value written into the topic """ result_topic_value = '' base_url = current_app.config['drupal-api-url'] json_api_path = '/jsonapi/node/record_set/' metadata_language_path = '/jsonapi/taxonomy_term/language_of_metadata/' user = current_app.config['drupal-user'] password = current_app.config['drupal-password'] auth = HTTPBasicAuth(user, password) headers = {'X-API-Key': current_app.config['drupal-api-key']} # Retrieve Drupal Entities for each language. # Returns default entity if there is a language missing. de_drupal_url = f'{base_url}/de{json_api_path}{record_set_id}' fr_drupal_url = f'{base_url}/fr{json_api_path}{record_set_id}' it_drupal_url = f'{base_url}/it{json_api_path}{record_set_id}' try: drupal_record_set_de = \ requests.get(de_drupal_url, headers=headers, auth=auth).json()['data'] drupal_record_set_fr = \ requests.get(fr_drupal_url, headers=headers, auth=auth).json()['data'] drupal_record_set_it = \ requests.get(it_drupal_url, headers=headers, auth=auth).json()['data'] institutions = drupal_record_set_de['relationships']['field_institution']['data'] institution_ids = UpdateRecordSet.get_institution_id_list(institutions, headers, auth) institutions = \ drupal_record_set_de['relationships']['field_resp_institution_access']['data'] access_institution_ids = UpdateRecordSet.get_institution_id_list( institutions, headers, auth) institutions = \ drupal_record_set_de['relationships']['field_resp_institution_master']['data'] master_institutions_ids = UpdateRecordSet.get_institution_id_list( institutions, headers, auth) institutions = \ drupal_record_set_de['relationships']['field_resp_institution_original']['data'] original_institution_ids = UpdateRecordSet.get_institution_id_list( institutions, headers, auth) metadata_language_codes = [] metadata_languages = \ drupal_record_set_de['relationships']['field_metadata_languages']['data'] for metadataLanguage in metadata_languages: drupal_record_set_language_code = \ requests.get(f'{base_url}{metadata_language_path}{metadataLanguage["id"]}', headers=headers, auth=auth) metadata_language_codes.append( drupal_record_set_language_code.json()['data']['attributes']['name'] ) related_record_sets_de = \ UpdateRecordSet.get_related_record_sets(drupal_record_set_de) related_record_sets_fr = \ UpdateRecordSet.get_related_record_sets(drupal_record_set_fr) related_record_sets_it = \ UpdateRecordSet.get_related_record_sets(drupal_record_set_it) result_topic_value = { 'type': drupal_record_set_de['type'], 'status': drupal_record_set_de['attributes']['status'], 'title_de': drupal_record_set_de['attributes']['title'], 'title_fr': drupal_record_set_fr['attributes']['title'], 'title_it': drupal_record_set_it['attributes']['title'], 'field_institution': institution_ids, 'field_metadata_language_codes': metadata_language_codes, 'computed_teaser_image_url': drupal_record_set_de['attributes']['computed_teaser_image_url'], 'field_processed_teaser_text_de': drupal_record_set_de['attributes']['field_processed_teaser_text'], 'field_processed_teaser_text_fr': drupal_record_set_fr['attributes']['field_processed_teaser_text'], 'field_processed_teaser_text_it': drupal_record_set_it['attributes']['field_processed_teaser_text'], 'field_old_memobase_id': drupal_record_set_de['attributes']['field_old_memobase_id'], 'field_access_de': drupal_record_set_de['attributes']['field_access'], 'field_access_fr': drupal_record_set_fr['attributes']['field_access'], 'field_access_it': drupal_record_set_it['attributes']['field_access'], 'field_access_memobase_de': drupal_record_set_de['attributes']['field_access_memobase'], 'field_access_memobase_fr': drupal_record_set_fr['attributes']['field_access_memobase'], 'field_access_memobase_it': drupal_record_set_it['attributes']['field_access_memobase'], 'field_content_de': drupal_record_set_de['attributes']['field_content'], 'field_content_fr': drupal_record_set_fr['attributes']['field_content'], 'field_content_it': drupal_record_set_it['attributes']['field_content'], 'field_context_de': drupal_record_set_de['attributes']['field_context'], 'field_context_fr': drupal_record_set_fr['attributes']['field_context'], 'field_context_it': drupal_record_set_it['attributes']['field_context'], 'field_data_transfer_de': drupal_record_set_de['attributes']['field_data_transfer'], 'field_data_transfer_fr': drupal_record_set_fr['attributes']['field_data_transfer'], 'field_data_transfer_it': drupal_record_set_it['attributes']['field_data_transfer'], 'field_documents_de': drupal_record_set_de['attributes']['field_documents'], 'field_documents_fr': drupal_record_set_fr['attributes']['field_documents'], 'field_documents_it': drupal_record_set_it['attributes']['field_documents'], 'field_info_on_development_de': drupal_record_set_de['attributes']['field_info_on_development'], 'field_info_on_development_fr': drupal_record_set_fr['attributes']['field_info_on_development'], 'field_info_on_development_it': drupal_record_set_it['attributes']['field_info_on_development'], 'field_language_de': drupal_record_set_de['attributes']['field_language'], 'field_language_fr': drupal_record_set_fr['attributes']['field_language'], 'field_language_it': drupal_record_set_it['attributes']['field_language'], 'field_memobase_id': drupal_record_set_de['attributes']['field_memobase_id'], 'field_notes': drupal_record_set_de['attributes']['field_notes'], 'field_original_description_de': drupal_record_set_de['attributes']['field_original_description'], 'field_original_description_fr': drupal_record_set_fr['attributes']['field_original_description'], 'field_original_description_it': drupal_record_set_it['attributes']['field_original_description'], 'field_original_id': drupal_record_set_de['attributes']['field_original_id'], 'field_original_shelf_mark': drupal_record_set_de['attributes']['field_original_shelf_mark'], 'field_original_title_de': drupal_record_set_de['attributes'][ 'field_original_title'], 'field_original_title_fr': drupal_record_set_fr['attributes']['field_original_title'], 'field_original_title_it': drupal_record_set_it['attributes']['field_original_title'], 'field_project_de': drupal_record_set_de['attributes']['field_project'], 'field_project_fr': drupal_record_set_fr['attributes']['field_project'], 'field_project_it': drupal_record_set_it['attributes']['field_project'], 'field_publications_de': drupal_record_set_de['attributes']['field_publications'], 'field_publications_fr': drupal_record_set_fr['attributes']['field_publications'], 'field_publications_it': drupal_record_set_it['attributes']['field_publications'], 'field_related_record_sets_de': related_record_sets_de, 'field_related_record_sets_fr': related_record_sets_fr, 'field_related_record_sets_it': related_record_sets_it, 'field_rights_de': drupal_record_set_de['attributes']['field_rights'], 'field_rights_fr': drupal_record_set_fr['attributes']['field_rights'], 'field_rights_it': drupal_record_set_it['attributes']['field_rights'], 'field_scope_de': drupal_record_set_de['attributes']['field_scope'], 'field_scope_fr': drupal_record_set_fr['attributes']['field_scope'], 'field_scope_it': drupal_record_set_it['attributes']['field_scope'], 'field_selection_de': drupal_record_set_de['attributes']['field_selection'], 'field_selection_fr': drupal_record_set_fr['attributes']['field_selection'], 'field_selection_it': drupal_record_set_it['attributes']['field_selection'], 'field_supported_by_memoriav': drupal_record_set_de['attributes']['field_supported_by_memoriav'], 'field_time_period': drupal_record_set_de['attributes']['field_time_period'], 'field_transfer_date': drupal_record_set_de['attributes']['field_transfer_date'], 'field_image_gallery': drupal_record_set_de['relationships']['field_image_gallery'], 'field_metadata_languages': metadata_language_codes, 'field_resp_institution_access': access_institution_ids, 'field_resp_institution_master': master_institutions_ids, 'field_resp_institution_original': original_institution_ids, 'field_teaser_image': drupal_record_set_de['relationships']['field_teaser_image'] } except LookupError as ex: msg = 'LookupError for ' + record_set_id + ': ' + str(ex) + '\n' + \ traceback.format_exc() + '\n' + \ 'baseRequest: ' + de_drupal_url + '\n' current_app.logger.error(msg) return { 'status': 'FAILURE', 'topic_key': result_topic_value.get('field_memobase_id'), 'result_topic_value': result_topic_value }, 500 except Exception as ex: msg = 'Exception for ' + record_set_id + ': ' + str(ex) + '\n' + \ traceback.format_exc() + '\n' + \ 'baseRequest: ' + de_drupal_url + '\n' current_app.logger.error(msg) return { 'status': 'FAILURE', 'topic_key': result_topic_value.get('field_memobase_id'), 'result_topic_value': result_topic_value, }, 500 # 2. write info into kafka topic try: producer = KafkaProducer(bootstrap_servers=current_app.config['kafka-broker-url'], value_serializer=lambda m: json.dumps(m, ensure_ascii=False) .encode('utf-8')) key = bytes(result_topic_value.get('field_memobase_id'), encoding='utf-8') producer.send(current_app.config['topic-drupal-export'], result_topic_value, key=key) except Exception as ex: msg = 'Exception for ' + record_set_id + ': ' + str(ex) + '\n' + \ traceback.format_exc() current_app.logger.error(msg) return { 'status': 'FAILURE', 'topic_key': result_topic_value.get('field_memobase_id'), 'result_topic_value': result_topic_value, 'exception': msg }, 500 current_app.logger.debug('success for ' + record_set_id) return { 'status': 'SUCCESS', 'topic_key': result_topic_value.get('field_memobase_id'), 'result_topic_value': result_topic_value }, 200 @staticmethod def get_institution_id_list(institution_data, headers, auth): institution_ids = [] for institution in institution_data: drupal_institution = \ requests.get(current_app.config['drupal-api-url'] + '/jsonapi/node/' + 'institution/' + institution['id'], headers=headers, auth=auth) institution_ids.append( drupal_institution.json()['data']['attributes']['field_memobase_id'] ) return institution_ids @staticmethod def get_related_record_sets(fields): related_record_sets = fields['attributes']['field_related_record_sets'] revised_related_record_sets = [] for relatedRecordSet in related_record_sets: if 'entity:node' in relatedRecordSet['uri']: relatedRecordSet['uri'] = fields['attributes']['field_memobase_id'] relatedRecordSet['title'] = 'internal' revised_related_record_sets.append(relatedRecordSet) return revised_related_record_sets