UpdateRecordSet.py 15.6 KB
Newer Older
1
2
import uuid

Jonas Waeber's avatar
Jonas Waeber committed
3
4
5
6
7
from flask_restful import Resource, current_app
from kafka import KafkaProducer
import requests
import json
import traceback
8
9

from kafka.errors import KafkaTimeoutError
Jonas Waeber's avatar
Jonas Waeber committed
10
11
12
13
from requests.auth import HTTPBasicAuth


class UpdateRecordSet(Resource):
14
15
16
17
18

    def __init__(self):
        self.producer = KafkaProducer(bootstrap_servers=current_app.config['kafka-broker-url'],
                                      value_serializer=lambda m: json.dumps(m, ensure_ascii=False)
                                      .encode('utf-8'))
19
20
21
22
23
24
25
26
        self.base_url = current_app.config['drupal-api-url']
        self.json_api_path = '/jsonapi/node/record_set/'
        self.institution_path = '/jsonapi/node/institution/'
        self.metadata_language_path = '/jsonapi/taxonomy_term/language_of_metadata/'
        self.headers = {'X-API-Key': current_app.config['drupal-api-key']}
        user = current_app.config['drupal-user']
        password = current_app.config['drupal-password']
        self.auth = HTTPBasicAuth(user, password)
27

Jonas Waeber's avatar
Jonas Waeber committed
28
    def get(self, record_set_drupal_uuid):
Jonas Waeber's avatar
Jonas Waeber committed
29
        """
30
        Update the record set with the given drupal UUID in the backend.
Jonas Waeber's avatar
Jonas Waeber committed
31
32
33
34
35
        ---
        tags:
          - Import Record Set
        parameters:
          - in: path
Jonas Waeber's avatar
Jonas Waeber committed
36
            name: record_set_drupal_uuid
Jonas Waeber's avatar
Jonas Waeber committed
37
38
39
40
41
42
43
44
45
46
47
48
49
50
            required: true
            description: The UUID of the updated recordset
            example: 0c4c777c-94f8-45ba-945a-bfe6967d40da
            type: string
        responses:
          200:
            description: Success, the information has been written into
                the kafka topic
            schema:
              properties:
                status:
                  type: string
                  example: SUCCESS/FAILURE
                  enum: ['SUCCESS', 'FAILURE']
Jonas Waeber's avatar
Jonas Waeber committed
51
                result_topic_value:
Jonas Waeber's avatar
Jonas Waeber committed
52
53
54
55
                  type: string/json
                  example: the value written into the topic

        """
Jonas Waeber's avatar
Jonas Waeber committed
56
57
        result_topic_value = ''

58
59
        # Retrieve Drupal Entities for each language.
        # Returns default entity if there is a language missing.
60
61
62
        de_drupal_url = f'{self.base_url}/de{self.json_api_path}{record_set_drupal_uuid}'
        fr_drupal_url = f'{self.base_url}/fr{self.json_api_path}{record_set_drupal_uuid}'
        it_drupal_url = f'{self.base_url}/it{self.json_api_path}{record_set_drupal_uuid}'
Jonas Waeber's avatar
Jonas Waeber committed
63
        try:
64
            drupal_record_set_de = \
65
                requests.get(de_drupal_url, headers=self.headers, auth=self.auth).json()['data']
Jonas Waeber's avatar
Jonas Waeber committed
66
            drupal_record_set_fr = \
67
                requests.get(fr_drupal_url, headers=self.headers, auth=self.auth).json()['data']
Jonas Waeber's avatar
Jonas Waeber committed
68
            drupal_record_set_it = \
69
                requests.get(it_drupal_url, headers=self.headers, auth=self.auth).json()['data']
Jonas Waeber's avatar
Jonas Waeber committed
70

71
            institutions = drupal_record_set_de['relationships']['field_institution']['data']
72
            institution_ids = UpdateRecordSet.get_institution_id_list(institutions)
Jonas Waeber's avatar
Jonas Waeber committed
73

74
75
            institutions = \
                drupal_record_set_de['relationships']['field_resp_institution_access']['data']
76
            access_institution_ids = UpdateRecordSet.get_institution_id_list(institutions)
Jonas Waeber's avatar
Jonas Waeber committed
77

78
79
            institutions = \
                drupal_record_set_de['relationships']['field_resp_institution_master']['data']
80
            master_institutions_ids = UpdateRecordSet.get_institution_id_list(institutions)
Jonas Waeber's avatar
Jonas Waeber committed
81
82

            institutions = \
83
                drupal_record_set_de['relationships']['field_resp_institution_original']['data']
84
            original_institution_ids = UpdateRecordSet.get_institution_id_list(institutions)
Jonas Waeber's avatar
Jonas Waeber committed
85
86
87

            metadata_language_codes = []
            metadata_languages = \
88
                drupal_record_set_de['relationships']['field_metadata_languages']['data']
Jonas Waeber's avatar
Jonas Waeber committed
89
90
            for metadataLanguage in metadata_languages:
                drupal_record_set_language_code = \
91
92
93
                    requests.get(
                        f'{self.base_url}{self.metadata_language_path}{metadataLanguage["id"]}',
                        headers=self.headers, auth=self.auth)
Jonas Waeber's avatar
Jonas Waeber committed
94
95
96
97
                metadata_language_codes.append(
                    drupal_record_set_language_code.json()['data']['attributes']['name']
                )

98
99
100
101
102
103
            related_record_sets_de = \
                UpdateRecordSet.get_related_record_sets(drupal_record_set_de)
            related_record_sets_fr = \
                UpdateRecordSet.get_related_record_sets(drupal_record_set_fr)
            related_record_sets_it = \
                UpdateRecordSet.get_related_record_sets(drupal_record_set_it)
Jonas Waeber's avatar
Jonas Waeber committed
104

Jonas Waeber's avatar
Jonas Waeber committed
105
            result_topic_value = {
106
107
108
                'type': drupal_record_set_de['type'],
                'status': drupal_record_set_de['attributes']['status'],
                'title_de': drupal_record_set_de['attributes']['title'],
Jonas Waeber's avatar
Jonas Waeber committed
109
110
111
112
113
                'title_fr': drupal_record_set_fr['attributes']['title'],
                'title_it': drupal_record_set_it['attributes']['title'],
                'field_institution': institution_ids,
                'field_metadata_language_codes': metadata_language_codes,
                'computed_teaser_image_url':
114
                    drupal_record_set_de['attributes']['computed_teaser_image_url'],
Jonas Waeber's avatar
Jonas Waeber committed
115
                'field_processed_teaser_text_de':
116
                    drupal_record_set_de['attributes']['field_processed_teaser_text'],
Jonas Waeber's avatar
Jonas Waeber committed
117
118
119
120
121
                'field_processed_teaser_text_fr':
                    drupal_record_set_fr['attributes']['field_processed_teaser_text'],
                'field_processed_teaser_text_it':
                    drupal_record_set_it['attributes']['field_processed_teaser_text'],
                'field_old_memobase_id':
122
123
                    drupal_record_set_de['attributes']['field_old_memobase_id'],
                'field_access_de': drupal_record_set_de['attributes']['field_access'],
Jonas Waeber's avatar
Jonas Waeber committed
124
125
126
                'field_access_fr': drupal_record_set_fr['attributes']['field_access'],
                'field_access_it': drupal_record_set_it['attributes']['field_access'],
                'field_access_memobase_de':
127
                    drupal_record_set_de['attributes']['field_access_memobase'],
Jonas Waeber's avatar
Jonas Waeber committed
128
129
130
131
                'field_access_memobase_fr':
                    drupal_record_set_fr['attributes']['field_access_memobase'],
                'field_access_memobase_it':
                    drupal_record_set_it['attributes']['field_access_memobase'],
132
                'field_content_de': drupal_record_set_de['attributes']['field_content'],
Jonas Waeber's avatar
Jonas Waeber committed
133
134
                'field_content_fr': drupal_record_set_fr['attributes']['field_content'],
                'field_content_it': drupal_record_set_it['attributes']['field_content'],
135
                'field_context_de': drupal_record_set_de['attributes']['field_context'],
Jonas Waeber's avatar
Jonas Waeber committed
136
137
                'field_context_fr': drupal_record_set_fr['attributes']['field_context'],
                'field_context_it': drupal_record_set_it['attributes']['field_context'],
138
                'field_data_transfer_de': drupal_record_set_de['attributes']['field_data_transfer'],
Jonas Waeber's avatar
Jonas Waeber committed
139
140
                'field_data_transfer_fr': drupal_record_set_fr['attributes']['field_data_transfer'],
                'field_data_transfer_it': drupal_record_set_it['attributes']['field_data_transfer'],
141
                'field_documents_de': drupal_record_set_de['attributes']['field_documents'],
Jonas Waeber's avatar
Jonas Waeber committed
142
143
144
                'field_documents_fr': drupal_record_set_fr['attributes']['field_documents'],
                'field_documents_it': drupal_record_set_it['attributes']['field_documents'],
                'field_info_on_development_de':
145
                    drupal_record_set_de['attributes']['field_info_on_development'],
Jonas Waeber's avatar
Jonas Waeber committed
146
147
148
149
                'field_info_on_development_fr':
                    drupal_record_set_fr['attributes']['field_info_on_development'],
                'field_info_on_development_it':
                    drupal_record_set_it['attributes']['field_info_on_development'],
150
                'field_language_de': drupal_record_set_de['attributes']['field_language'],
Jonas Waeber's avatar
Jonas Waeber committed
151
152
                'field_language_fr': drupal_record_set_fr['attributes']['field_language'],
                'field_language_it': drupal_record_set_it['attributes']['field_language'],
153
154
                'field_memobase_id': drupal_record_set_de['attributes']['field_memobase_id'],
                'field_notes': drupal_record_set_de['attributes']['field_notes'],
Jonas Waeber's avatar
Jonas Waeber committed
155
                'field_original_description_de':
156
                    drupal_record_set_de['attributes']['field_original_description'],
Jonas Waeber's avatar
Jonas Waeber committed
157
158
159
160
                'field_original_description_fr':
                    drupal_record_set_fr['attributes']['field_original_description'],
                'field_original_description_it':
                    drupal_record_set_it['attributes']['field_original_description'],
161
                'field_original_id': drupal_record_set_de['attributes']['field_original_id'],
Jonas Waeber's avatar
Jonas Waeber committed
162
                'field_original_shelf_mark':
163
164
165
                    drupal_record_set_de['attributes']['field_original_shelf_mark'],
                'field_original_title_de': drupal_record_set_de['attributes'][
                    'field_original_title'],
Jonas Waeber's avatar
Jonas Waeber committed
166
167
168
169
                'field_original_title_fr':
                    drupal_record_set_fr['attributes']['field_original_title'],
                'field_original_title_it':
                    drupal_record_set_it['attributes']['field_original_title'],
170
                'field_project_de': drupal_record_set_de['attributes']['field_project'],
Jonas Waeber's avatar
Jonas Waeber committed
171
172
                'field_project_fr': drupal_record_set_fr['attributes']['field_project'],
                'field_project_it': drupal_record_set_it['attributes']['field_project'],
173
                'field_publications_de': drupal_record_set_de['attributes']['field_publications'],
Jonas Waeber's avatar
Jonas Waeber committed
174
175
                'field_publications_fr': drupal_record_set_fr['attributes']['field_publications'],
                'field_publications_it': drupal_record_set_it['attributes']['field_publications'],
176
177
178
179
                'field_related_record_sets_de': related_record_sets_de,
                'field_related_record_sets_fr': related_record_sets_fr,
                'field_related_record_sets_it': related_record_sets_it,
                'field_rights_de': drupal_record_set_de['attributes']['field_rights'],
Jonas Waeber's avatar
Jonas Waeber committed
180
181
                'field_rights_fr': drupal_record_set_fr['attributes']['field_rights'],
                'field_rights_it': drupal_record_set_it['attributes']['field_rights'],
182
                'field_scope_de': drupal_record_set_de['attributes']['field_scope'],
Jonas Waeber's avatar
Jonas Waeber committed
183
184
                'field_scope_fr': drupal_record_set_fr['attributes']['field_scope'],
                'field_scope_it': drupal_record_set_it['attributes']['field_scope'],
185
                'field_selection_de': drupal_record_set_de['attributes']['field_selection'],
Jonas Waeber's avatar
Jonas Waeber committed
186
187
188
                'field_selection_fr': drupal_record_set_fr['attributes']['field_selection'],
                'field_selection_it': drupal_record_set_it['attributes']['field_selection'],
                'field_supported_by_memoriav':
189
190
191
                    drupal_record_set_de['attributes']['field_supported_by_memoriav'],
                'field_time_period': drupal_record_set_de['attributes']['field_time_period'],
                'field_transfer_date': drupal_record_set_de['attributes']['field_transfer_date'],
Jonas Waeber's avatar
Jonas Waeber committed
192
                'field_image_gallery':
193
                    drupal_record_set_de['relationships']['field_image_gallery'],
Jonas Waeber's avatar
Jonas Waeber committed
194
                'field_metadata_languages': metadata_language_codes,
195
196
197
198
                'field_resp_institution_access': access_institution_ids,
                'field_resp_institution_master': master_institutions_ids,
                'field_resp_institution_original': original_institution_ids,
                'field_teaser_image': drupal_record_set_de['relationships']['field_teaser_image']
Jonas Waeber's avatar
Jonas Waeber committed
199
200
            }
        except LookupError as ex:
Jonas Waeber's avatar
Jonas Waeber committed
201
            msg = 'LookupError for ' + record_set_drupal_uuid + ': ' + str(ex) + '\n' + \
Jonas Waeber's avatar
Jonas Waeber committed
202
                  traceback.format_exc() + '\n' + \
203
                  'baseRequest: ' + de_drupal_url + '\n'
Jonas Waeber's avatar
Jonas Waeber committed
204
205
206
            current_app.logger.error(msg)
            return {
                       'status': 'FAILURE',
Jonas Waeber's avatar
Jonas Waeber committed
207
208
                       'topic_key': result_topic_value.get('field_memobase_id'),
                       'result_topic_value': result_topic_value
Jonas Waeber's avatar
Jonas Waeber committed
209
210
                   }, 500
        except Exception as ex:
Jonas Waeber's avatar
Jonas Waeber committed
211
            msg = 'Exception for ' + record_set_drupal_uuid + ': ' + str(ex) + '\n' + \
Jonas Waeber's avatar
Jonas Waeber committed
212
                  traceback.format_exc() + '\n' + \
213
                  'baseRequest: ' + de_drupal_url + '\n'
Jonas Waeber's avatar
Jonas Waeber committed
214
215
216
            current_app.logger.error(msg)
            return {
                       'status': 'FAILURE',
Jonas Waeber's avatar
Jonas Waeber committed
217
218
                       'topic_key': result_topic_value.get('field_memobase_id'),
                       'result_topic_value': result_topic_value,
Jonas Waeber's avatar
Jonas Waeber committed
219
                   }, 500
Jonas Waeber's avatar
Jonas Waeber committed
220

Jonas Waeber's avatar
Jonas Waeber committed
221
        return self.send_message(result_topic_value, record_set_drupal_uuid)
Jonas Waeber's avatar
Jonas Waeber committed
222

223
    def send_message(self, result_topic_value, record_set_drupal_id):
224
        headers = [
225
            ('recordSetId', bytes(result_topic_value.get('field_memobase_id'), encoding='utf-8')),
Jonas Waeber's avatar
Jonas Waeber committed
226
            ('sessionId', bytes(str(uuid.uuid4()), encoding='utf-8')),
Jonas Waeber's avatar
Jonas Waeber committed
227
            ('institutionId', bytes('none', encoding='utf-8')),
228
            ('isPublished', bytes(str(result_topic_value['status']), encoding='utf-8'))
229
        ]
Jonas Waeber's avatar
Jonas Waeber committed
230
        try:
Jonas Waeber's avatar
Jonas Waeber committed
231
            key = bytes(result_topic_value.get('field_memobase_id'), encoding='utf-8')
Jonas Waeber's avatar
Jonas Waeber committed
232
233
234
            current_app.logger.debug(
                f'Send message: key={key}, headers={headers}, '
                f'message: {json.dumps(result_topic_value, ensure_ascii=False)}')
235
236
237
            self.producer.send(current_app.config['topic-drupal-export'], result_topic_value,
                               key=key, headers=headers)
        except KafkaTimeoutError as ex:
238
            msg = f'KafkaTimeoutError ({record_set_drupal_id}): {ex}.'
239
240
241
242
243
244
245
            current_app.logger.error(msg)
            return {
                       'status': 'FAILURE',
                       'topic_key': result_topic_value.get('field_memobase_id'),
                       'result_topic_value': result_topic_value,
                       'exception': msg
                   }, 503
Jonas Waeber's avatar
Jonas Waeber committed
246
        except Exception as ex:
Jonas Waeber's avatar
Jonas Waeber committed
247
            msg = f'Could not import {result_topic_value.get("field_memobase_id")} ' \
248
                  f'(Drupal UUID: {record_set_drupal_id}) (Unknown Exception): ' + str(ex)
Jonas Waeber's avatar
Jonas Waeber committed
249
            current_app.logger.error(f"{msg}\n{traceback.format_exc()}")
Jonas Waeber's avatar
Jonas Waeber committed
250
251
            return {
                       'status': 'FAILURE',
Jonas Waeber's avatar
Jonas Waeber committed
252
253
                       'topic_key': result_topic_value.get('field_memobase_id'),
                       'result_topic_value': result_topic_value,
Jonas Waeber's avatar
Jonas Waeber committed
254
                       'exception': msg
255
                   }, 503
Jonas Waeber's avatar
Jonas Waeber committed
256

257
        current_app.logger.debug('success for ' + record_set_drupal_id)
Jonas Waeber's avatar
Jonas Waeber committed
258
259
        return {
                   'status': 'SUCCESS',
Jonas Waeber's avatar
Jonas Waeber committed
260
261
                   'topic_key': result_topic_value.get('field_memobase_id'),
                   'result_topic_value': result_topic_value
Jonas Waeber's avatar
Jonas Waeber committed
262
263
               }, 200

264
    def get_institution_id_list(self, institution_data):
Jonas Waeber's avatar
Jonas Waeber committed
265
266
        institution_ids = []
        for institution in institution_data:
267
268
            url = f"{self.base_url}{self.institution_path}{institution['id']}"
            drupal_institution = requests.get(url, headers=self.headers, auth=self.auth)
Jonas Waeber's avatar
Jonas Waeber committed
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
            institution_ids.append(
                drupal_institution.json()['data']['attributes']['field_memobase_id']
            )
        return institution_ids

    @staticmethod
    def get_related_record_sets(fields):
        related_record_sets = fields['attributes']['field_related_record_sets']
        revised_related_record_sets = []
        for relatedRecordSet in related_record_sets:
            if 'entity:node' in relatedRecordSet['uri']:
                relatedRecordSet['uri'] = fields['attributes']['field_memobase_id']
                relatedRecordSet['title'] = 'internal'
            revised_related_record_sets.append(relatedRecordSet)
        return revised_related_record_sets