update_record_set.py 16.3 KB
Newer Older
Jonas Waeber's avatar
Jonas Waeber committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
# Import API Service
# Copyright (C) 2020-2021 Memobase Project
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <https://www.gnu.org/licenses/>.
16
17
import uuid

Jonas Waeber's avatar
Jonas Waeber committed
18
19
20
21
22
from flask_restful import Resource, current_app
from kafka import KafkaProducer
import requests
import json
import traceback
23
24

from kafka.errors import KafkaTimeoutError
Jonas Waeber's avatar
Jonas Waeber committed
25
26
27
28
from requests.auth import HTTPBasicAuth


class UpdateRecordSet(Resource):
29
30
31
32
33

    def __init__(self):
        self.producer = KafkaProducer(bootstrap_servers=current_app.config['kafka-broker-url'],
                                      value_serializer=lambda m: json.dumps(m, ensure_ascii=False)
                                      .encode('utf-8'))
34
35
36
37
38
39
40
41
        self.base_url = current_app.config['drupal-api-url']
        self.json_api_path = '/jsonapi/node/record_set/'
        self.institution_path = '/jsonapi/node/institution/'
        self.metadata_language_path = '/jsonapi/taxonomy_term/language_of_metadata/'
        self.headers = {'X-API-Key': current_app.config['drupal-api-key']}
        user = current_app.config['drupal-user']
        password = current_app.config['drupal-password']
        self.auth = HTTPBasicAuth(user, password)
42

Jonas Waeber's avatar
Jonas Waeber committed
43
    def get(self, record_set_drupal_uuid):
Jonas Waeber's avatar
Jonas Waeber committed
44
        """
45
        Update the record set with the given drupal UUID in the backend.
Jonas Waeber's avatar
Jonas Waeber committed
46
47
48
49
50
        ---
        tags:
          - Import Record Set
        parameters:
          - in: path
Jonas Waeber's avatar
Jonas Waeber committed
51
            name: record_set_drupal_uuid
Jonas Waeber's avatar
Jonas Waeber committed
52
53
54
55
56
57
58
59
60
61
62
63
64
65
            required: true
            description: The UUID of the updated recordset
            example: 0c4c777c-94f8-45ba-945a-bfe6967d40da
            type: string
        responses:
          200:
            description: Success, the information has been written into
                the kafka topic
            schema:
              properties:
                status:
                  type: string
                  example: SUCCESS/FAILURE
                  enum: ['SUCCESS', 'FAILURE']
Jonas Waeber's avatar
Jonas Waeber committed
66
                result_topic_value:
Jonas Waeber's avatar
Jonas Waeber committed
67
68
69
70
                  type: string/json
                  example: the value written into the topic

        """
Jonas Waeber's avatar
Jonas Waeber committed
71
72
        result_topic_value = ''

73
74
        # Retrieve Drupal Entities for each language.
        # Returns default entity if there is a language missing.
75
76
77
        de_drupal_url = f'{self.base_url}/de{self.json_api_path}{record_set_drupal_uuid}'
        fr_drupal_url = f'{self.base_url}/fr{self.json_api_path}{record_set_drupal_uuid}'
        it_drupal_url = f'{self.base_url}/it{self.json_api_path}{record_set_drupal_uuid}'
Jonas Waeber's avatar
Jonas Waeber committed
78
        try:
79
            drupal_record_set_de = \
80
                requests.get(de_drupal_url, headers=self.headers, auth=self.auth).json()['data']
Jonas Waeber's avatar
Jonas Waeber committed
81
            drupal_record_set_fr = \
82
                requests.get(fr_drupal_url, headers=self.headers, auth=self.auth).json()['data']
Jonas Waeber's avatar
Jonas Waeber committed
83
            drupal_record_set_it = \
84
                requests.get(it_drupal_url, headers=self.headers, auth=self.auth).json()['data']
Jonas Waeber's avatar
Jonas Waeber committed
85

86
            institutions = drupal_record_set_de['relationships']['field_institution']['data']
Jonas Waeber's avatar
Jonas Waeber committed
87
            institution_ids = self.get_institution_id_list(institutions)
Jonas Waeber's avatar
Jonas Waeber committed
88

89
90
            institutions = \
                drupal_record_set_de['relationships']['field_resp_institution_access']['data']
Jonas Waeber's avatar
Jonas Waeber committed
91
            access_institution_ids = self.get_institution_id_list(institutions)
Jonas Waeber's avatar
Jonas Waeber committed
92

93
94
            institutions = \
                drupal_record_set_de['relationships']['field_resp_institution_master']['data']
Jonas Waeber's avatar
Jonas Waeber committed
95
            master_institutions_ids = self.get_institution_id_list(institutions)
Jonas Waeber's avatar
Jonas Waeber committed
96
97

            institutions = \
98
                drupal_record_set_de['relationships']['field_resp_institution_original']['data']
Jonas Waeber's avatar
Jonas Waeber committed
99
            original_institution_ids = self.get_institution_id_list(institutions)
Jonas Waeber's avatar
Jonas Waeber committed
100
101
102

            metadata_language_codes = []
            metadata_languages = \
103
                drupal_record_set_de['relationships']['field_metadata_languages']['data']
Jonas Waeber's avatar
Jonas Waeber committed
104
105
            for metadataLanguage in metadata_languages:
                drupal_record_set_language_code = \
106
107
108
                    requests.get(
                        f'{self.base_url}{self.metadata_language_path}{metadataLanguage["id"]}',
                        headers=self.headers, auth=self.auth)
Jonas Waeber's avatar
Jonas Waeber committed
109
110
111
112
                metadata_language_codes.append(
                    drupal_record_set_language_code.json()['data']['attributes']['name']
                )

113
114
115
116
117
118
            related_record_sets_de = \
                UpdateRecordSet.get_related_record_sets(drupal_record_set_de)
            related_record_sets_fr = \
                UpdateRecordSet.get_related_record_sets(drupal_record_set_fr)
            related_record_sets_it = \
                UpdateRecordSet.get_related_record_sets(drupal_record_set_it)
Jonas Waeber's avatar
Jonas Waeber committed
119

Jonas Waeber's avatar
Jonas Waeber committed
120
            result_topic_value = {
121
122
123
                'type': drupal_record_set_de['type'],
                'status': drupal_record_set_de['attributes']['status'],
                'title_de': drupal_record_set_de['attributes']['title'],
Jonas Waeber's avatar
Jonas Waeber committed
124
125
126
127
128
                'title_fr': drupal_record_set_fr['attributes']['title'],
                'title_it': drupal_record_set_it['attributes']['title'],
                'field_institution': institution_ids,
                'field_metadata_language_codes': metadata_language_codes,
                'computed_teaser_image_url':
129
                    drupal_record_set_de['attributes']['computed_teaser_image_url'],
Jonas Waeber's avatar
Jonas Waeber committed
130
                'field_processed_teaser_text_de':
131
                    drupal_record_set_de['attributes']['field_processed_teaser_text'],
Jonas Waeber's avatar
Jonas Waeber committed
132
133
134
135
136
                'field_processed_teaser_text_fr':
                    drupal_record_set_fr['attributes']['field_processed_teaser_text'],
                'field_processed_teaser_text_it':
                    drupal_record_set_it['attributes']['field_processed_teaser_text'],
                'field_old_memobase_id':
137
138
                    drupal_record_set_de['attributes']['field_old_memobase_id'],
                'field_access_de': drupal_record_set_de['attributes']['field_access'],
Jonas Waeber's avatar
Jonas Waeber committed
139
140
141
                'field_access_fr': drupal_record_set_fr['attributes']['field_access'],
                'field_access_it': drupal_record_set_it['attributes']['field_access'],
                'field_access_memobase_de':
142
                    drupal_record_set_de['attributes']['field_access_memobase'],
Jonas Waeber's avatar
Jonas Waeber committed
143
144
145
146
                'field_access_memobase_fr':
                    drupal_record_set_fr['attributes']['field_access_memobase'],
                'field_access_memobase_it':
                    drupal_record_set_it['attributes']['field_access_memobase'],
147
                'field_content_de': drupal_record_set_de['attributes']['field_content'],
Jonas Waeber's avatar
Jonas Waeber committed
148
149
                'field_content_fr': drupal_record_set_fr['attributes']['field_content'],
                'field_content_it': drupal_record_set_it['attributes']['field_content'],
150
                'field_context_de': drupal_record_set_de['attributes']['field_context'],
Jonas Waeber's avatar
Jonas Waeber committed
151
152
                'field_context_fr': drupal_record_set_fr['attributes']['field_context'],
                'field_context_it': drupal_record_set_it['attributes']['field_context'],
153
                'field_data_transfer_de': drupal_record_set_de['attributes']['field_data_transfer'],
Jonas Waeber's avatar
Jonas Waeber committed
154
155
                'field_data_transfer_fr': drupal_record_set_fr['attributes']['field_data_transfer'],
                'field_data_transfer_it': drupal_record_set_it['attributes']['field_data_transfer'],
156
                'field_documents_de': drupal_record_set_de['attributes']['field_documents'],
Jonas Waeber's avatar
Jonas Waeber committed
157
158
159
                'field_documents_fr': drupal_record_set_fr['attributes']['field_documents'],
                'field_documents_it': drupal_record_set_it['attributes']['field_documents'],
                'field_info_on_development_de':
160
                    drupal_record_set_de['attributes']['field_info_on_development'],
Jonas Waeber's avatar
Jonas Waeber committed
161
162
163
164
                'field_info_on_development_fr':
                    drupal_record_set_fr['attributes']['field_info_on_development'],
                'field_info_on_development_it':
                    drupal_record_set_it['attributes']['field_info_on_development'],
165
                'field_language_de': drupal_record_set_de['attributes']['field_language'],
Jonas Waeber's avatar
Jonas Waeber committed
166
167
                'field_language_fr': drupal_record_set_fr['attributes']['field_language'],
                'field_language_it': drupal_record_set_it['attributes']['field_language'],
168
169
                'field_memobase_id': drupal_record_set_de['attributes']['field_memobase_id'],
                'field_notes': drupal_record_set_de['attributes']['field_notes'],
Jonas Waeber's avatar
Jonas Waeber committed
170
                'field_original_description_de':
171
                    drupal_record_set_de['attributes']['field_original_description'],
Jonas Waeber's avatar
Jonas Waeber committed
172
173
174
175
                'field_original_description_fr':
                    drupal_record_set_fr['attributes']['field_original_description'],
                'field_original_description_it':
                    drupal_record_set_it['attributes']['field_original_description'],
176
                'field_original_id': drupal_record_set_de['attributes']['field_original_id'],
Jonas Waeber's avatar
Jonas Waeber committed
177
                'field_original_shelf_mark':
178
179
180
                    drupal_record_set_de['attributes']['field_original_shelf_mark'],
                'field_original_title_de': drupal_record_set_de['attributes'][
                    'field_original_title'],
Jonas Waeber's avatar
Jonas Waeber committed
181
182
183
184
                'field_original_title_fr':
                    drupal_record_set_fr['attributes']['field_original_title'],
                'field_original_title_it':
                    drupal_record_set_it['attributes']['field_original_title'],
185
                'field_project_de': drupal_record_set_de['attributes']['field_project'],
Jonas Waeber's avatar
Jonas Waeber committed
186
187
                'field_project_fr': drupal_record_set_fr['attributes']['field_project'],
                'field_project_it': drupal_record_set_it['attributes']['field_project'],
188
                'field_publications_de': drupal_record_set_de['attributes']['field_publications'],
Jonas Waeber's avatar
Jonas Waeber committed
189
190
                'field_publications_fr': drupal_record_set_fr['attributes']['field_publications'],
                'field_publications_it': drupal_record_set_it['attributes']['field_publications'],
191
192
193
194
                'field_related_record_sets_de': related_record_sets_de,
                'field_related_record_sets_fr': related_record_sets_fr,
                'field_related_record_sets_it': related_record_sets_it,
                'field_rights_de': drupal_record_set_de['attributes']['field_rights'],
Jonas Waeber's avatar
Jonas Waeber committed
195
196
                'field_rights_fr': drupal_record_set_fr['attributes']['field_rights'],
                'field_rights_it': drupal_record_set_it['attributes']['field_rights'],
197
                'field_scope_de': drupal_record_set_de['attributes']['field_scope'],
Jonas Waeber's avatar
Jonas Waeber committed
198
199
                'field_scope_fr': drupal_record_set_fr['attributes']['field_scope'],
                'field_scope_it': drupal_record_set_it['attributes']['field_scope'],
200
                'field_selection_de': drupal_record_set_de['attributes']['field_selection'],
Jonas Waeber's avatar
Jonas Waeber committed
201
202
203
                'field_selection_fr': drupal_record_set_fr['attributes']['field_selection'],
                'field_selection_it': drupal_record_set_it['attributes']['field_selection'],
                'field_supported_by_memoriav':
204
205
206
                    drupal_record_set_de['attributes']['field_supported_by_memoriav'],
                'field_time_period': drupal_record_set_de['attributes']['field_time_period'],
                'field_transfer_date': drupal_record_set_de['attributes']['field_transfer_date'],
Jonas Waeber's avatar
Jonas Waeber committed
207
                'field_image_gallery':
208
                    drupal_record_set_de['relationships']['field_image_gallery'],
Jonas Waeber's avatar
Jonas Waeber committed
209
                'field_metadata_languages': metadata_language_codes,
210
211
212
213
                'field_resp_institution_access': access_institution_ids,
                'field_resp_institution_master': master_institutions_ids,
                'field_resp_institution_original': original_institution_ids,
                'field_teaser_image': drupal_record_set_de['relationships']['field_teaser_image']
Jonas Waeber's avatar
Jonas Waeber committed
214
215
            }
        except LookupError as ex:
Jonas Waeber's avatar
Jonas Waeber committed
216
            msg = 'LookupError for ' + record_set_drupal_uuid + ': ' + str(ex) + '\n' + \
Jonas Waeber's avatar
Jonas Waeber committed
217
                  traceback.format_exc() + '\n' + \
218
                  'baseRequest: ' + de_drupal_url + '\n'
Jonas Waeber's avatar
Jonas Waeber committed
219
220
221
            current_app.logger.error(msg)
            return {
                       'status': 'FAILURE',
Jonas Waeber's avatar
Jonas Waeber committed
222
223
                       'topic_key': result_topic_value.get('field_memobase_id'),
                       'result_topic_value': result_topic_value
Jonas Waeber's avatar
Jonas Waeber committed
224
225
                   }, 500
        except Exception as ex:
Jonas Waeber's avatar
Jonas Waeber committed
226
            msg = 'Exception for ' + record_set_drupal_uuid + ': ' + str(ex) + '\n' + \
Jonas Waeber's avatar
Jonas Waeber committed
227
                  traceback.format_exc() + '\n' + \
228
                  'baseRequest: ' + de_drupal_url + '\n'
Jonas Waeber's avatar
Jonas Waeber committed
229
230
231
            current_app.logger.error(msg)
            return {
                       'status': 'FAILURE',
Jonas Waeber's avatar
Jonas Waeber committed
232
233
                       'topic_key': result_topic_value.get('field_memobase_id'),
                       'result_topic_value': result_topic_value,
Jonas Waeber's avatar
Jonas Waeber committed
234
                   }, 500
Jonas Waeber's avatar
Jonas Waeber committed
235

Jonas Waeber's avatar
Jonas Waeber committed
236
        return self.send_message(result_topic_value, record_set_drupal_uuid)
Jonas Waeber's avatar
Jonas Waeber committed
237

238
    def send_message(self, result_topic_value, record_set_drupal_id):
239
        headers = [
240
            ('recordSetId', bytes(result_topic_value.get('field_memobase_id'), encoding='utf-8')),
Jonas Waeber's avatar
Jonas Waeber committed
241
            ('sessionId', bytes(str(uuid.uuid4()), encoding='utf-8')),
Jonas Waeber's avatar
Jonas Waeber committed
242
            ('institutionId', bytes('none', encoding='utf-8')),
243
            ('isPublished', bytes(str(result_topic_value['status']), encoding='utf-8'))
244
        ]
Jonas Waeber's avatar
Jonas Waeber committed
245
        try:
Jonas Waeber's avatar
Jonas Waeber committed
246
            key = bytes(result_topic_value.get('field_memobase_id'), encoding='utf-8')
Jonas Waeber's avatar
Jonas Waeber committed
247
248
249
            current_app.logger.debug(
                f'Send message: key={key}, headers={headers}, '
                f'message: {json.dumps(result_topic_value, ensure_ascii=False)}')
250
251
252
            self.producer.send(current_app.config['topic-drupal-export'], result_topic_value,
                               key=key, headers=headers)
        except KafkaTimeoutError as ex:
253
            msg = f'KafkaTimeoutError ({record_set_drupal_id}): {ex}.'
254
255
256
257
258
259
260
            current_app.logger.error(msg)
            return {
                       'status': 'FAILURE',
                       'topic_key': result_topic_value.get('field_memobase_id'),
                       'result_topic_value': result_topic_value,
                       'exception': msg
                   }, 503
Jonas Waeber's avatar
Jonas Waeber committed
261
        except Exception as ex:
Jonas Waeber's avatar
Jonas Waeber committed
262
            msg = f'Could not import {result_topic_value.get("field_memobase_id")} ' \
263
                  f'(Drupal UUID: {record_set_drupal_id}) (Unknown Exception): ' + str(ex)
Jonas Waeber's avatar
Jonas Waeber committed
264
            current_app.logger.error(f"{msg}\n{traceback.format_exc()}")
Jonas Waeber's avatar
Jonas Waeber committed
265
266
            return {
                       'status': 'FAILURE',
Jonas Waeber's avatar
Jonas Waeber committed
267
268
                       'topic_key': result_topic_value.get('field_memobase_id'),
                       'result_topic_value': result_topic_value,
Jonas Waeber's avatar
Jonas Waeber committed
269
                       'exception': msg
270
                   }, 503
Jonas Waeber's avatar
Jonas Waeber committed
271

272
        current_app.logger.debug('success for ' + record_set_drupal_id)
Jonas Waeber's avatar
Jonas Waeber committed
273
274
        return {
                   'status': 'SUCCESS',
Jonas Waeber's avatar
Jonas Waeber committed
275
276
                   'topic_key': result_topic_value.get('field_memobase_id'),
                   'result_topic_value': result_topic_value
Jonas Waeber's avatar
Jonas Waeber committed
277
278
               }, 200

279
    def get_institution_id_list(self, institution_data):
Jonas Waeber's avatar
Jonas Waeber committed
280
281
        institution_ids = []
        for institution in institution_data:
282
283
            url = f"{self.base_url}{self.institution_path}{institution['id']}"
            drupal_institution = requests.get(url, headers=self.headers, auth=self.auth)
Jonas Waeber's avatar
Jonas Waeber committed
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
            institution_ids.append(
                drupal_institution.json()['data']['attributes']['field_memobase_id']
            )
        return institution_ids

    @staticmethod
    def get_related_record_sets(fields):
        related_record_sets = fields['attributes']['field_related_record_sets']
        revised_related_record_sets = []
        for relatedRecordSet in related_record_sets:
            if 'entity:node' in relatedRecordSet['uri']:
                relatedRecordSet['uri'] = fields['attributes']['field_memobase_id']
                relatedRecordSet['title'] = 'internal'
            revised_related_record_sets.append(relatedRecordSet)
        return revised_related_record_sets