UpdateRecordSet.py 15.6 KB
Newer Older
1
2
import uuid

Jonas Waeber's avatar
Jonas Waeber committed
3
4
5
6
7
from flask_restful import Resource, current_app
from kafka import KafkaProducer
import requests
import json
import traceback
8
9

from kafka.errors import KafkaTimeoutError
Jonas Waeber's avatar
Jonas Waeber committed
10
11
12
13
from requests.auth import HTTPBasicAuth


class UpdateRecordSet(Resource):
14
15
16
17
18

    def __init__(self):
        self.producer = KafkaProducer(bootstrap_servers=current_app.config['kafka-broker-url'],
                                      value_serializer=lambda m: json.dumps(m, ensure_ascii=False)
                                      .encode('utf-8'))
19

Jonas Waeber's avatar
Jonas Waeber committed
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
    # Todo write/correct comment for swagger
    def get(self, record_set_id):
        """
        Get uuid of updated recordset form drupal, gets more information form \
            drupal and puts it into the dedicated kafka topic
        ---
        tags:
          - Import Record Set
        parameters:
          - in: path
            name: record_set_id
            required: true
            description: The UUID of the updated recordset
            example: 0c4c777c-94f8-45ba-945a-bfe6967d40da
            type: string
        responses:
          200:
            description: Success, the information has been written into
                the kafka topic
            schema:
              properties:
                status:
                  type: string
                  example: SUCCESS/FAILURE
                  enum: ['SUCCESS', 'FAILURE']
Jonas Waeber's avatar
Jonas Waeber committed
45
                result_topic_value:
Jonas Waeber's avatar
Jonas Waeber committed
46
47
48
49
                  type: string/json
                  example: the value written into the topic

        """
Jonas Waeber's avatar
Jonas Waeber committed
50
51
52
        result_topic_value = ''
        base_url = current_app.config['drupal-api-url']
        json_api_path = '/jsonapi/node/record_set/'
53
        metadata_language_path = '/jsonapi/taxonomy_term/language_of_metadata/'
Jonas Waeber's avatar
Jonas Waeber committed
54
55
56
57
        user = current_app.config['drupal-user']
        password = current_app.config['drupal-password']
        auth = HTTPBasicAuth(user, password)
        headers = {'X-API-Key': current_app.config['drupal-api-key']}
Jonas Waeber's avatar
Jonas Waeber committed
58

59
60
61
62
63
        # Retrieve Drupal Entities for each language.
        # Returns default entity if there is a language missing.
        de_drupal_url = f'{base_url}/de{json_api_path}{record_set_id}'
        fr_drupal_url = f'{base_url}/fr{json_api_path}{record_set_id}'
        it_drupal_url = f'{base_url}/it{json_api_path}{record_set_id}'
Jonas Waeber's avatar
Jonas Waeber committed
64
        try:
65
66
            drupal_record_set_de = \
                requests.get(de_drupal_url, headers=headers, auth=auth).json()['data']
Jonas Waeber's avatar
Jonas Waeber committed
67
            drupal_record_set_fr = \
68
                requests.get(fr_drupal_url, headers=headers, auth=auth).json()['data']
Jonas Waeber's avatar
Jonas Waeber committed
69
            drupal_record_set_it = \
70
                requests.get(it_drupal_url, headers=headers, auth=auth).json()['data']
Jonas Waeber's avatar
Jonas Waeber committed
71

72
73
            institutions = drupal_record_set_de['relationships']['field_institution']['data']
            institution_ids = UpdateRecordSet.get_institution_id_list(institutions, headers, auth)
Jonas Waeber's avatar
Jonas Waeber committed
74

75
76
77
            institutions = \
                drupal_record_set_de['relationships']['field_resp_institution_access']['data']
            access_institution_ids = UpdateRecordSet.get_institution_id_list(
Jonas Waeber's avatar
Jonas Waeber committed
78
79
                institutions, headers, auth)

80
81
82
            institutions = \
                drupal_record_set_de['relationships']['field_resp_institution_master']['data']
            master_institutions_ids = UpdateRecordSet.get_institution_id_list(
Jonas Waeber's avatar
Jonas Waeber committed
83
84
85
                institutions, headers, auth)

            institutions = \
86
87
                drupal_record_set_de['relationships']['field_resp_institution_original']['data']
            original_institution_ids = UpdateRecordSet.get_institution_id_list(
Jonas Waeber's avatar
Jonas Waeber committed
88
89
90
91
                institutions, headers, auth)

            metadata_language_codes = []
            metadata_languages = \
92
                drupal_record_set_de['relationships']['field_metadata_languages']['data']
Jonas Waeber's avatar
Jonas Waeber committed
93
94
            for metadataLanguage in metadata_languages:
                drupal_record_set_language_code = \
95
                    requests.get(f'{base_url}{metadata_language_path}{metadataLanguage["id"]}',
Jonas Waeber's avatar
Jonas Waeber committed
96
97
98
99
100
                                 headers=headers, auth=auth)
                metadata_language_codes.append(
                    drupal_record_set_language_code.json()['data']['attributes']['name']
                )

101
102
103
104
105
106
            related_record_sets_de = \
                UpdateRecordSet.get_related_record_sets(drupal_record_set_de)
            related_record_sets_fr = \
                UpdateRecordSet.get_related_record_sets(drupal_record_set_fr)
            related_record_sets_it = \
                UpdateRecordSet.get_related_record_sets(drupal_record_set_it)
Jonas Waeber's avatar
Jonas Waeber committed
107

Jonas Waeber's avatar
Jonas Waeber committed
108
            result_topic_value = {
109
110
111
                'type': drupal_record_set_de['type'],
                'status': drupal_record_set_de['attributes']['status'],
                'title_de': drupal_record_set_de['attributes']['title'],
Jonas Waeber's avatar
Jonas Waeber committed
112
113
114
115
116
                'title_fr': drupal_record_set_fr['attributes']['title'],
                'title_it': drupal_record_set_it['attributes']['title'],
                'field_institution': institution_ids,
                'field_metadata_language_codes': metadata_language_codes,
                'computed_teaser_image_url':
117
                    drupal_record_set_de['attributes']['computed_teaser_image_url'],
Jonas Waeber's avatar
Jonas Waeber committed
118
                'field_processed_teaser_text_de':
119
                    drupal_record_set_de['attributes']['field_processed_teaser_text'],
Jonas Waeber's avatar
Jonas Waeber committed
120
121
122
123
124
                'field_processed_teaser_text_fr':
                    drupal_record_set_fr['attributes']['field_processed_teaser_text'],
                'field_processed_teaser_text_it':
                    drupal_record_set_it['attributes']['field_processed_teaser_text'],
                'field_old_memobase_id':
125
126
                    drupal_record_set_de['attributes']['field_old_memobase_id'],
                'field_access_de': drupal_record_set_de['attributes']['field_access'],
Jonas Waeber's avatar
Jonas Waeber committed
127
128
129
                'field_access_fr': drupal_record_set_fr['attributes']['field_access'],
                'field_access_it': drupal_record_set_it['attributes']['field_access'],
                'field_access_memobase_de':
130
                    drupal_record_set_de['attributes']['field_access_memobase'],
Jonas Waeber's avatar
Jonas Waeber committed
131
132
133
134
                'field_access_memobase_fr':
                    drupal_record_set_fr['attributes']['field_access_memobase'],
                'field_access_memobase_it':
                    drupal_record_set_it['attributes']['field_access_memobase'],
135
                'field_content_de': drupal_record_set_de['attributes']['field_content'],
Jonas Waeber's avatar
Jonas Waeber committed
136
137
                'field_content_fr': drupal_record_set_fr['attributes']['field_content'],
                'field_content_it': drupal_record_set_it['attributes']['field_content'],
138
                'field_context_de': drupal_record_set_de['attributes']['field_context'],
Jonas Waeber's avatar
Jonas Waeber committed
139
140
                'field_context_fr': drupal_record_set_fr['attributes']['field_context'],
                'field_context_it': drupal_record_set_it['attributes']['field_context'],
141
                'field_data_transfer_de': drupal_record_set_de['attributes']['field_data_transfer'],
Jonas Waeber's avatar
Jonas Waeber committed
142
143
                'field_data_transfer_fr': drupal_record_set_fr['attributes']['field_data_transfer'],
                'field_data_transfer_it': drupal_record_set_it['attributes']['field_data_transfer'],
144
                'field_documents_de': drupal_record_set_de['attributes']['field_documents'],
Jonas Waeber's avatar
Jonas Waeber committed
145
146
147
                'field_documents_fr': drupal_record_set_fr['attributes']['field_documents'],
                'field_documents_it': drupal_record_set_it['attributes']['field_documents'],
                'field_info_on_development_de':
148
                    drupal_record_set_de['attributes']['field_info_on_development'],
Jonas Waeber's avatar
Jonas Waeber committed
149
150
151
152
                'field_info_on_development_fr':
                    drupal_record_set_fr['attributes']['field_info_on_development'],
                'field_info_on_development_it':
                    drupal_record_set_it['attributes']['field_info_on_development'],
153
                'field_language_de': drupal_record_set_de['attributes']['field_language'],
Jonas Waeber's avatar
Jonas Waeber committed
154
155
                'field_language_fr': drupal_record_set_fr['attributes']['field_language'],
                'field_language_it': drupal_record_set_it['attributes']['field_language'],
156
157
                'field_memobase_id': drupal_record_set_de['attributes']['field_memobase_id'],
                'field_notes': drupal_record_set_de['attributes']['field_notes'],
Jonas Waeber's avatar
Jonas Waeber committed
158
                'field_original_description_de':
159
                    drupal_record_set_de['attributes']['field_original_description'],
Jonas Waeber's avatar
Jonas Waeber committed
160
161
162
163
                'field_original_description_fr':
                    drupal_record_set_fr['attributes']['field_original_description'],
                'field_original_description_it':
                    drupal_record_set_it['attributes']['field_original_description'],
164
                'field_original_id': drupal_record_set_de['attributes']['field_original_id'],
Jonas Waeber's avatar
Jonas Waeber committed
165
                'field_original_shelf_mark':
166
167
168
                    drupal_record_set_de['attributes']['field_original_shelf_mark'],
                'field_original_title_de': drupal_record_set_de['attributes'][
                    'field_original_title'],
Jonas Waeber's avatar
Jonas Waeber committed
169
170
171
172
                'field_original_title_fr':
                    drupal_record_set_fr['attributes']['field_original_title'],
                'field_original_title_it':
                    drupal_record_set_it['attributes']['field_original_title'],
173
                'field_project_de': drupal_record_set_de['attributes']['field_project'],
Jonas Waeber's avatar
Jonas Waeber committed
174
175
                'field_project_fr': drupal_record_set_fr['attributes']['field_project'],
                'field_project_it': drupal_record_set_it['attributes']['field_project'],
176
                'field_publications_de': drupal_record_set_de['attributes']['field_publications'],
Jonas Waeber's avatar
Jonas Waeber committed
177
178
                'field_publications_fr': drupal_record_set_fr['attributes']['field_publications'],
                'field_publications_it': drupal_record_set_it['attributes']['field_publications'],
179
180
181
182
                'field_related_record_sets_de': related_record_sets_de,
                'field_related_record_sets_fr': related_record_sets_fr,
                'field_related_record_sets_it': related_record_sets_it,
                'field_rights_de': drupal_record_set_de['attributes']['field_rights'],
Jonas Waeber's avatar
Jonas Waeber committed
183
184
                'field_rights_fr': drupal_record_set_fr['attributes']['field_rights'],
                'field_rights_it': drupal_record_set_it['attributes']['field_rights'],
185
                'field_scope_de': drupal_record_set_de['attributes']['field_scope'],
Jonas Waeber's avatar
Jonas Waeber committed
186
187
                'field_scope_fr': drupal_record_set_fr['attributes']['field_scope'],
                'field_scope_it': drupal_record_set_it['attributes']['field_scope'],
188
                'field_selection_de': drupal_record_set_de['attributes']['field_selection'],
Jonas Waeber's avatar
Jonas Waeber committed
189
190
191
                'field_selection_fr': drupal_record_set_fr['attributes']['field_selection'],
                'field_selection_it': drupal_record_set_it['attributes']['field_selection'],
                'field_supported_by_memoriav':
192
193
194
                    drupal_record_set_de['attributes']['field_supported_by_memoriav'],
                'field_time_period': drupal_record_set_de['attributes']['field_time_period'],
                'field_transfer_date': drupal_record_set_de['attributes']['field_transfer_date'],
Jonas Waeber's avatar
Jonas Waeber committed
195
                'field_image_gallery':
196
                    drupal_record_set_de['relationships']['field_image_gallery'],
Jonas Waeber's avatar
Jonas Waeber committed
197
                'field_metadata_languages': metadata_language_codes,
198
199
200
201
                'field_resp_institution_access': access_institution_ids,
                'field_resp_institution_master': master_institutions_ids,
                'field_resp_institution_original': original_institution_ids,
                'field_teaser_image': drupal_record_set_de['relationships']['field_teaser_image']
Jonas Waeber's avatar
Jonas Waeber committed
202
203
204
205
            }
        except LookupError as ex:
            msg = 'LookupError for ' + record_set_id + ': ' + str(ex) + '\n' + \
                  traceback.format_exc() + '\n' + \
206
                  'baseRequest: ' + de_drupal_url + '\n'
Jonas Waeber's avatar
Jonas Waeber committed
207
208
209
            current_app.logger.error(msg)
            return {
                       'status': 'FAILURE',
Jonas Waeber's avatar
Jonas Waeber committed
210
211
                       'topic_key': result_topic_value.get('field_memobase_id'),
                       'result_topic_value': result_topic_value
Jonas Waeber's avatar
Jonas Waeber committed
212
213
214
215
                   }, 500
        except Exception as ex:
            msg = 'Exception for ' + record_set_id + ': ' + str(ex) + '\n' + \
                  traceback.format_exc() + '\n' + \
216
                  'baseRequest: ' + de_drupal_url + '\n'
Jonas Waeber's avatar
Jonas Waeber committed
217
218
219
            current_app.logger.error(msg)
            return {
                       'status': 'FAILURE',
Jonas Waeber's avatar
Jonas Waeber committed
220
221
                       'topic_key': result_topic_value.get('field_memobase_id'),
                       'result_topic_value': result_topic_value,
Jonas Waeber's avatar
Jonas Waeber committed
222
                   }, 500
Jonas Waeber's avatar
Jonas Waeber committed
223
224

        return self.send_message(result_topic_value, record_set_id)
Jonas Waeber's avatar
Jonas Waeber committed
225

226
227
    def send_message(self, result_topic_value, record_set_id):
        headers = [
Jonas Waeber's avatar
Jonas Waeber committed
228
            ('recordSetId', bytes(record_set_id, encoding='utf-8')),
Jonas Waeber's avatar
Jonas Waeber committed
229
            ('sessionId', uuid.uuid4().bytes),
Jonas Waeber's avatar
Jonas Waeber committed
230
            ('institutionId', bytes('none', encoding='utf-8')),
231
            ('isPublished', bytes(str(result_topic_value['status']), encoding='utf-8'))
232
        ]
Jonas Waeber's avatar
Jonas Waeber committed
233
        try:
Jonas Waeber's avatar
Jonas Waeber committed
234
            key = bytes(result_topic_value.get('field_memobase_id'), encoding='utf-8')
Jonas Waeber's avatar
Jonas Waeber committed
235
236
237
            current_app.logger.debug(
                f'Send message: key={key}, headers={headers}, '
                f'message: {json.dumps(result_topic_value, ensure_ascii=False)}')
238
239
240
241
242
243
244
245
246
247
248
249
            self.producer.send(current_app.config['topic-drupal-export'], result_topic_value,
                               key=key, headers=headers)

        except KafkaTimeoutError as ex:
            msg = f'KafkaTimeoutError ({record_set_id}): {ex}.'
            current_app.logger.error(msg)
            return {
                       'status': 'FAILURE',
                       'topic_key': result_topic_value.get('field_memobase_id'),
                       'result_topic_value': result_topic_value,
                       'exception': msg
                   }, 503
Jonas Waeber's avatar
Jonas Waeber committed
250
        except Exception as ex:
Jonas Waeber's avatar
Jonas Waeber committed
251
252
            msg = f'Could not import {result_topic_value.get("field_memobase_id")} ' \
                  f'(Drupal UUID: {record_set_id}) (Unknown Exception): ' + str(ex)
Jonas Waeber's avatar
Jonas Waeber committed
253
            current_app.logger.error(f"{msg}\n{traceback.format_exc()}")
Jonas Waeber's avatar
Jonas Waeber committed
254
255
            return {
                       'status': 'FAILURE',
Jonas Waeber's avatar
Jonas Waeber committed
256
257
                       'topic_key': result_topic_value.get('field_memobase_id'),
                       'result_topic_value': result_topic_value,
Jonas Waeber's avatar
Jonas Waeber committed
258
                       'exception': msg
259
                   }, 503
Jonas Waeber's avatar
Jonas Waeber committed
260
261
262
263

        current_app.logger.debug('success for ' + record_set_id)
        return {
                   'status': 'SUCCESS',
Jonas Waeber's avatar
Jonas Waeber committed
264
265
                   'topic_key': result_topic_value.get('field_memobase_id'),
                   'result_topic_value': result_topic_value
Jonas Waeber's avatar
Jonas Waeber committed
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
               }, 200

    @staticmethod
    def get_institution_id_list(institution_data, headers, auth):
        institution_ids = []
        for institution in institution_data:
            drupal_institution = \
                requests.get(current_app.config['drupal-api-url'] + '/jsonapi/node/' +
                             'institution/' + institution['id'], headers=headers, auth=auth)
            institution_ids.append(
                drupal_institution.json()['data']['attributes']['field_memobase_id']
            )
        return institution_ids

    @staticmethod
    def get_related_record_sets(fields):
        related_record_sets = fields['attributes']['field_related_record_sets']
        revised_related_record_sets = []
        for relatedRecordSet in related_record_sets:
            if 'entity:node' in relatedRecordSet['uri']:
                relatedRecordSet['uri'] = fields['attributes']['field_memobase_id']
                relatedRecordSet['title'] = 'internal'
            revised_related_record_sets.append(relatedRecordSet)
        return revised_related_record_sets