UpdateRecordSet.py

from flask_restful import Resource, current_app
from kafka import KafkaProducer
import requests
import json
import traceback
from requests.auth import HTTPBasicAuth


class UpdateRecordSet(Resource):

    # Todo write/correct comment for swagger
    def get(self, record_set_id):
        """
        Get uuid of updated recordset form drupal, gets more information form \
            drupal and puts it into the dedicated kafka topic
        ---
        tags:
          - Import Record Set
        parameters:
          - in: path
            name: record_set_id
            required: true
            description: The UUID of the updated recordset
            example: 0c4c777c-94f8-45ba-945a-bfe6967d40da
            type: string
        responses:
          200:
            description: Success, the information has been written into
                the kafka topic
            schema:
              properties:
                status:
                  type: string
                  example: SUCCESS/FAILURE
                  enum: ['SUCCESS', 'FAILURE']
                result_topic_value:
                  type: string/json
                  example: the value written into the topic

        """
        result_topic_value = ''
        base_url = current_app.config['drupal-api-url']
        json_api_path = '/jsonapi/node/record_set/'
        metadata_language_path = '/jsonapi/taxonomy_term/language_of_metadata/'
        user = current_app.config['drupal-user']
        password = current_app.config['drupal-password']
        auth = HTTPBasicAuth(user, password)
        headers = {'X-API-Key': current_app.config['drupal-api-key']}

        # Retrieve Drupal Entities for each language.
        # Returns default entity if there is a language missing.
        de_drupal_url = f'{base_url}/de{json_api_path}{record_set_id}'
        fr_drupal_url = f'{base_url}/fr{json_api_path}{record_set_id}'
        it_drupal_url = f'{base_url}/it{json_api_path}{record_set_id}'
        try:
            drupal_record_set_de = \
                requests.get(de_drupal_url, headers=headers, auth=auth).json()['data']
            drupal_record_set_fr = \
                requests.get(fr_drupal_url, headers=headers, auth=auth).json()['data']
            drupal_record_set_it = \
                requests.get(it_drupal_url, headers=headers, auth=auth).json()['data']

            institutions = drupal_record_set_de['relationships']['field_institution']['data']
            institution_ids = UpdateRecordSet.get_institution_id_list(institutions, headers, auth)

            institutions = \
                drupal_record_set_de['relationships']['field_resp_institution_access']['data']
            access_institution_ids = UpdateRecordSet.get_institution_id_list(
                institutions, headers, auth)

            institutions = \
                drupal_record_set_de['relationships']['field_resp_institution_master']['data']
            master_institutions_ids = UpdateRecordSet.get_institution_id_list(
                institutions, headers, auth)

            institutions = \
                drupal_record_set_de['relationships']['field_resp_institution_original']['data']
            original_institution_ids = UpdateRecordSet.get_institution_id_list(
                institutions, headers, auth)

            metadata_language_codes = []
            metadata_languages = \
                drupal_record_set_de['relationships']['field_metadata_languages']['data']
            for metadataLanguage in metadata_languages:
                drupal_record_set_language_code = \
                    requests.get(f'{base_url}{metadata_language_path}{metadataLanguage["id"]}',
                                 headers=headers, auth=auth)
                metadata_language_codes.append(
                    drupal_record_set_language_code.json()['data']['attributes']['name']
                )

            related_record_sets_de = \
                UpdateRecordSet.get_related_record_sets(drupal_record_set_de)
            related_record_sets_fr = \
                UpdateRecordSet.get_related_record_sets(drupal_record_set_fr)
            related_record_sets_it = \
                UpdateRecordSet.get_related_record_sets(drupal_record_set_it)

            result_topic_value = {
                'type': drupal_record_set_de['type'],
                'status': drupal_record_set_de['attributes']['status'],
                'title_de': drupal_record_set_de['attributes']['title'],
                'title_fr': drupal_record_set_fr['attributes']['title'],
                'title_it': drupal_record_set_it['attributes']['title'],
                'field_institution': institution_ids,
                'field_metadata_language_codes': metadata_language_codes,
                'computed_teaser_image_url':
                    drupal_record_set_de['attributes']['computed_teaser_image_url'],
                'field_processed_teaser_text_de':
                    drupal_record_set_de['attributes']['field_processed_teaser_text'],
                'field_processed_teaser_text_fr':
                    drupal_record_set_fr['attributes']['field_processed_teaser_text'],
                'field_processed_teaser_text_it':
                    drupal_record_set_it['attributes']['field_processed_teaser_text'],
                'field_old_memobase_id':
                    drupal_record_set_de['attributes']['field_old_memobase_id'],
                'field_access_de': drupal_record_set_de['attributes']['field_access'],
                'field_access_fr': drupal_record_set_fr['attributes']['field_access'],
                'field_access_it': drupal_record_set_it['attributes']['field_access'],
                'field_access_memobase_de':
                    drupal_record_set_de['attributes']['field_access_memobase'],
                'field_access_memobase_fr':
                    drupal_record_set_fr['attributes']['field_access_memobase'],
                'field_access_memobase_it':
                    drupal_record_set_it['attributes']['field_access_memobase'],
                'field_content_de': drupal_record_set_de['attributes']['field_content'],
                'field_content_fr': drupal_record_set_fr['attributes']['field_content'],
                'field_content_it': drupal_record_set_it['attributes']['field_content'],
                'field_context_de': drupal_record_set_de['attributes']['field_context'],
                'field_context_fr': drupal_record_set_fr['attributes']['field_context'],
                'field_context_it': drupal_record_set_it['attributes']['field_context'],
                'field_data_transfer_de': drupal_record_set_de['attributes']['field_data_transfer'],
                'field_data_transfer_fr': drupal_record_set_fr['attributes']['field_data_transfer'],
                'field_data_transfer_it': drupal_record_set_it['attributes']['field_data_transfer'],
                'field_documents_de': drupal_record_set_de['attributes']['field_documents'],
                'field_documents_fr': drupal_record_set_fr['attributes']['field_documents'],
                'field_documents_it': drupal_record_set_it['attributes']['field_documents'],
                'field_info_on_development_de':
                    drupal_record_set_de['attributes']['field_info_on_development'],
                'field_info_on_development_fr':
                    drupal_record_set_fr['attributes']['field_info_on_development'],
                'field_info_on_development_it':
                    drupal_record_set_it['attributes']['field_info_on_development'],
                'field_language_de': drupal_record_set_de['attributes']['field_language'],
                'field_language_fr': drupal_record_set_fr['attributes']['field_language'],
                'field_language_it': drupal_record_set_it['attributes']['field_language'],
                'field_memobase_id': drupal_record_set_de['attributes']['field_memobase_id'],
                'field_notes': drupal_record_set_de['attributes']['field_notes'],
                'field_original_description_de':
                    drupal_record_set_de['attributes']['field_original_description'],
                'field_original_description_fr':
                    drupal_record_set_fr['attributes']['field_original_description'],
                'field_original_description_it':
                    drupal_record_set_it['attributes']['field_original_description'],
                'field_original_id': drupal_record_set_de['attributes']['field_original_id'],
                'field_original_shelf_mark':
                    drupal_record_set_de['attributes']['field_original_shelf_mark'],
                'field_original_title_de': drupal_record_set_de['attributes'][
                    'field_original_title'],
                'field_original_title_fr':
                    drupal_record_set_fr['attributes']['field_original_title'],
                'field_original_title_it':
                    drupal_record_set_it['attributes']['field_original_title'],
                'field_project_de': drupal_record_set_de['attributes']['field_project'],
                'field_project_fr': drupal_record_set_fr['attributes']['field_project'],
                'field_project_it': drupal_record_set_it['attributes']['field_project'],
                'field_publications_de': drupal_record_set_de['attributes']['field_publications'],
                'field_publications_fr': drupal_record_set_fr['attributes']['field_publications'],
                'field_publications_it': drupal_record_set_it['attributes']['field_publications'],
                'field_related_record_sets_de': related_record_sets_de,
                'field_related_record_sets_fr': related_record_sets_fr,
                'field_related_record_sets_it': related_record_sets_it,
                'field_rights_de': drupal_record_set_de['attributes']['field_rights'],
                'field_rights_fr': drupal_record_set_fr['attributes']['field_rights'],
                'field_rights_it': drupal_record_set_it['attributes']['field_rights'],
                'field_scope_de': drupal_record_set_de['attributes']['field_scope'],
                'field_scope_fr': drupal_record_set_fr['attributes']['field_scope'],
                'field_scope_it': drupal_record_set_it['attributes']['field_scope'],
                'field_selection_de': drupal_record_set_de['attributes']['field_selection'],
                'field_selection_fr': drupal_record_set_fr['attributes']['field_selection'],
                'field_selection_it': drupal_record_set_it['attributes']['field_selection'],
                'field_supported_by_memoriav':
                    drupal_record_set_de['attributes']['field_supported_by_memoriav'],
                'field_time_period': drupal_record_set_de['attributes']['field_time_period'],
                'field_transfer_date': drupal_record_set_de['attributes']['field_transfer_date'],
                'field_image_gallery':
                    drupal_record_set_de['relationships']['field_image_gallery'],
                'field_metadata_languages': metadata_language_codes,
                'field_resp_institution_access': access_institution_ids,
                'field_resp_institution_master': master_institutions_ids,
                'field_resp_institution_original': original_institution_ids,
                'field_teaser_image': drupal_record_set_de['relationships']['field_teaser_image']
            }
        except LookupError as ex:
            msg = 'LookupError for ' + record_set_id + ': ' + str(ex) + '\n' + \
                  traceback.format_exc() + '\n' + \
                  'baseRequest: ' + de_drupal_url + '\n'
            current_app.logger.error(msg)
            return {
                       'status': 'FAILURE',
                       'topic_key': result_topic_value.get('field_memobase_id'),
                       'result_topic_value': result_topic_value
                   }, 500
        except Exception as ex:
            msg = 'Exception for ' + record_set_id + ': ' + str(ex) + '\n' + \
                  traceback.format_exc() + '\n' + \
                  'baseRequest: ' + de_drupal_url + '\n'
            current_app.logger.error(msg)
            return {
                       'status': 'FAILURE',
                       'topic_key': result_topic_value.get('field_memobase_id'),
                       'result_topic_value': result_topic_value,
                   }, 500

        # 2. write info into kafka topic
        try:
            producer = KafkaProducer(bootstrap_servers=current_app.config['kafka-broker-url'],
                                     value_serializer=lambda m: json.dumps(m, ensure_ascii=False)
                                     .encode('utf-8'))
            key = bytes(result_topic_value.get('field_memobase_id'), encoding='utf-8')
            producer.send(current_app.config['topic-drupal-export'], result_topic_value, key=key)
        except Exception as ex:
            msg = 'Exception for ' + record_set_id + ': ' + str(ex) + '\n' + \
                  traceback.format_exc()
            current_app.logger.error(msg)
            return {
                       'status': 'FAILURE',
                       'topic_key': result_topic_value.get('field_memobase_id'),
                       'result_topic_value': result_topic_value,
                       'exception': msg
                   }, 500

        current_app.logger.debug('success for ' + record_set_id)
        return {
                   'status': 'SUCCESS',
                   'topic_key': result_topic_value.get('field_memobase_id'),
                   'result_topic_value': result_topic_value
               }, 200

    @staticmethod
    def get_institution_id_list(institution_data, headers, auth):
        institution_ids = []
        for institution in institution_data:
            drupal_institution = \
                requests.get(current_app.config['drupal-api-url'] + '/jsonapi/node/' +
                             'institution/' + institution['id'], headers=headers, auth=auth)
            institution_ids.append(
                drupal_institution.json()['data']['attributes']['field_memobase_id']
            )
        return institution_ids

    @staticmethod
    def get_related_record_sets(fields):
        related_record_sets = fields['attributes']['field_related_record_sets']
        revised_related_record_sets = []
        for relatedRecordSet in related_record_sets:
            if 'entity:node' in relatedRecordSet['uri']:
                relatedRecordSet['uri'] = fields['attributes']['field_memobase_id']
                relatedRecordSet['title'] = 'internal'
            revised_related_record_sets.append(relatedRecordSet)
        return revised_related_record_sets