UpdateRecordSet.py 14.4 KB
Newer Older
Jonas Waeber's avatar
Jonas Waeber committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
from flask_restful import Resource, current_app
from kafka import KafkaProducer
import requests
import json
import traceback
from requests.auth import HTTPBasicAuth


class UpdateRecordSet(Resource):

    # Todo write/correct comment for swagger
    def get(self, record_set_id):
        """
        Get uuid of updated recordset form drupal, gets more information form \
            drupal and puts it into the dedicated kafka topic
        ---
        tags:
          - Import Record Set
        parameters:
          - in: path
            name: record_set_id
            required: true
            description: The UUID of the updated recordset
            example: 0c4c777c-94f8-45ba-945a-bfe6967d40da
            type: string
        responses:
          200:
            description: Success, the information has been written into
                the kafka topic
            schema:
              properties:
                status:
                  type: string
                  example: SUCCESS/FAILURE
                  enum: ['SUCCESS', 'FAILURE']
Jonas Waeber's avatar
Jonas Waeber committed
36
                result_topic_value:
Jonas Waeber's avatar
Jonas Waeber committed
37
38
39
40
                  type: string/json
                  example: the value written into the topic

        """
Jonas Waeber's avatar
Jonas Waeber committed
41
42
43
        result_topic_value = ''
        base_url = current_app.config['drupal-api-url']
        json_api_path = '/jsonapi/node/record_set/'
44
        metadata_language_path = '/jsonapi/taxonomy_term/language_of_metadata/'
Jonas Waeber's avatar
Jonas Waeber committed
45
46
47
48
        user = current_app.config['drupal-user']
        password = current_app.config['drupal-password']
        auth = HTTPBasicAuth(user, password)
        headers = {'X-API-Key': current_app.config['drupal-api-key']}
Jonas Waeber's avatar
Jonas Waeber committed
49

50
51
52
53
54
        # Retrieve Drupal Entities for each language.
        # Returns default entity if there is a language missing.
        de_drupal_url = f'{base_url}/de{json_api_path}{record_set_id}'
        fr_drupal_url = f'{base_url}/fr{json_api_path}{record_set_id}'
        it_drupal_url = f'{base_url}/it{json_api_path}{record_set_id}'
Jonas Waeber's avatar
Jonas Waeber committed
55
        try:
56
57
            drupal_record_set_de = \
                requests.get(de_drupal_url, headers=headers, auth=auth).json()['data']
Jonas Waeber's avatar
Jonas Waeber committed
58
            drupal_record_set_fr = \
59
                requests.get(fr_drupal_url, headers=headers, auth=auth).json()['data']
Jonas Waeber's avatar
Jonas Waeber committed
60
            drupal_record_set_it = \
61
                requests.get(it_drupal_url, headers=headers, auth=auth).json()['data']
Jonas Waeber's avatar
Jonas Waeber committed
62

63
64
            institutions = drupal_record_set_de['relationships']['field_institution']['data']
            institution_ids = UpdateRecordSet.get_institution_id_list(institutions, headers, auth)
Jonas Waeber's avatar
Jonas Waeber committed
65

66
67
68
            institutions = \
                drupal_record_set_de['relationships']['field_resp_institution_access']['data']
            access_institution_ids = UpdateRecordSet.get_institution_id_list(
Jonas Waeber's avatar
Jonas Waeber committed
69
70
                institutions, headers, auth)

71
72
73
            institutions = \
                drupal_record_set_de['relationships']['field_resp_institution_master']['data']
            master_institutions_ids = UpdateRecordSet.get_institution_id_list(
Jonas Waeber's avatar
Jonas Waeber committed
74
75
76
                institutions, headers, auth)

            institutions = \
77
78
                drupal_record_set_de['relationships']['field_resp_institution_original']['data']
            original_institution_ids = UpdateRecordSet.get_institution_id_list(
Jonas Waeber's avatar
Jonas Waeber committed
79
80
81
82
                institutions, headers, auth)

            metadata_language_codes = []
            metadata_languages = \
83
                drupal_record_set_de['relationships']['field_metadata_languages']['data']
Jonas Waeber's avatar
Jonas Waeber committed
84
85
            for metadataLanguage in metadata_languages:
                drupal_record_set_language_code = \
86
                    requests.get(f'{base_url}{metadata_language_path}{metadataLanguage["id"]}',
Jonas Waeber's avatar
Jonas Waeber committed
87
88
89
90
91
                                 headers=headers, auth=auth)
                metadata_language_codes.append(
                    drupal_record_set_language_code.json()['data']['attributes']['name']
                )

92
93
94
95
96
97
            related_record_sets_de = \
                UpdateRecordSet.get_related_record_sets(drupal_record_set_de)
            related_record_sets_fr = \
                UpdateRecordSet.get_related_record_sets(drupal_record_set_fr)
            related_record_sets_it = \
                UpdateRecordSet.get_related_record_sets(drupal_record_set_it)
Jonas Waeber's avatar
Jonas Waeber committed
98

Jonas Waeber's avatar
Jonas Waeber committed
99
            result_topic_value = {
100
101
102
                'type': drupal_record_set_de['type'],
                'status': drupal_record_set_de['attributes']['status'],
                'title_de': drupal_record_set_de['attributes']['title'],
Jonas Waeber's avatar
Jonas Waeber committed
103
104
105
106
107
                'title_fr': drupal_record_set_fr['attributes']['title'],
                'title_it': drupal_record_set_it['attributes']['title'],
                'field_institution': institution_ids,
                'field_metadata_language_codes': metadata_language_codes,
                'computed_teaser_image_url':
108
                    drupal_record_set_de['attributes']['computed_teaser_image_url'],
Jonas Waeber's avatar
Jonas Waeber committed
109
                'field_processed_teaser_text_de':
110
                    drupal_record_set_de['attributes']['field_processed_teaser_text'],
Jonas Waeber's avatar
Jonas Waeber committed
111
112
113
114
115
                'field_processed_teaser_text_fr':
                    drupal_record_set_fr['attributes']['field_processed_teaser_text'],
                'field_processed_teaser_text_it':
                    drupal_record_set_it['attributes']['field_processed_teaser_text'],
                'field_old_memobase_id':
116
117
                    drupal_record_set_de['attributes']['field_old_memobase_id'],
                'field_access_de': drupal_record_set_de['attributes']['field_access'],
Jonas Waeber's avatar
Jonas Waeber committed
118
119
120
                'field_access_fr': drupal_record_set_fr['attributes']['field_access'],
                'field_access_it': drupal_record_set_it['attributes']['field_access'],
                'field_access_memobase_de':
121
                    drupal_record_set_de['attributes']['field_access_memobase'],
Jonas Waeber's avatar
Jonas Waeber committed
122
123
124
125
                'field_access_memobase_fr':
                    drupal_record_set_fr['attributes']['field_access_memobase'],
                'field_access_memobase_it':
                    drupal_record_set_it['attributes']['field_access_memobase'],
126
                'field_content_de': drupal_record_set_de['attributes']['field_content'],
Jonas Waeber's avatar
Jonas Waeber committed
127
128
                'field_content_fr': drupal_record_set_fr['attributes']['field_content'],
                'field_content_it': drupal_record_set_it['attributes']['field_content'],
129
                'field_context_de': drupal_record_set_de['attributes']['field_context'],
Jonas Waeber's avatar
Jonas Waeber committed
130
131
                'field_context_fr': drupal_record_set_fr['attributes']['field_context'],
                'field_context_it': drupal_record_set_it['attributes']['field_context'],
132
                'field_data_transfer_de': drupal_record_set_de['attributes']['field_data_transfer'],
Jonas Waeber's avatar
Jonas Waeber committed
133
134
                'field_data_transfer_fr': drupal_record_set_fr['attributes']['field_data_transfer'],
                'field_data_transfer_it': drupal_record_set_it['attributes']['field_data_transfer'],
135
                'field_documents_de': drupal_record_set_de['attributes']['field_documents'],
Jonas Waeber's avatar
Jonas Waeber committed
136
137
138
                'field_documents_fr': drupal_record_set_fr['attributes']['field_documents'],
                'field_documents_it': drupal_record_set_it['attributes']['field_documents'],
                'field_info_on_development_de':
139
                    drupal_record_set_de['attributes']['field_info_on_development'],
Jonas Waeber's avatar
Jonas Waeber committed
140
141
142
143
                'field_info_on_development_fr':
                    drupal_record_set_fr['attributes']['field_info_on_development'],
                'field_info_on_development_it':
                    drupal_record_set_it['attributes']['field_info_on_development'],
144
                'field_language_de': drupal_record_set_de['attributes']['field_language'],
Jonas Waeber's avatar
Jonas Waeber committed
145
146
                'field_language_fr': drupal_record_set_fr['attributes']['field_language'],
                'field_language_it': drupal_record_set_it['attributes']['field_language'],
147
148
                'field_memobase_id': drupal_record_set_de['attributes']['field_memobase_id'],
                'field_notes': drupal_record_set_de['attributes']['field_notes'],
Jonas Waeber's avatar
Jonas Waeber committed
149
                'field_original_description_de':
150
                    drupal_record_set_de['attributes']['field_original_description'],
Jonas Waeber's avatar
Jonas Waeber committed
151
152
153
154
                'field_original_description_fr':
                    drupal_record_set_fr['attributes']['field_original_description'],
                'field_original_description_it':
                    drupal_record_set_it['attributes']['field_original_description'],
155
                'field_original_id': drupal_record_set_de['attributes']['field_original_id'],
Jonas Waeber's avatar
Jonas Waeber committed
156
                'field_original_shelf_mark':
157
158
159
                    drupal_record_set_de['attributes']['field_original_shelf_mark'],
                'field_original_title_de': drupal_record_set_de['attributes'][
                    'field_original_title'],
Jonas Waeber's avatar
Jonas Waeber committed
160
161
162
163
                'field_original_title_fr':
                    drupal_record_set_fr['attributes']['field_original_title'],
                'field_original_title_it':
                    drupal_record_set_it['attributes']['field_original_title'],
164
                'field_project_de': drupal_record_set_de['attributes']['field_project'],
Jonas Waeber's avatar
Jonas Waeber committed
165
166
                'field_project_fr': drupal_record_set_fr['attributes']['field_project'],
                'field_project_it': drupal_record_set_it['attributes']['field_project'],
167
                'field_publications_de': drupal_record_set_de['attributes']['field_publications'],
Jonas Waeber's avatar
Jonas Waeber committed
168
169
                'field_publications_fr': drupal_record_set_fr['attributes']['field_publications'],
                'field_publications_it': drupal_record_set_it['attributes']['field_publications'],
170
171
172
173
                'field_related_record_sets_de': related_record_sets_de,
                'field_related_record_sets_fr': related_record_sets_fr,
                'field_related_record_sets_it': related_record_sets_it,
                'field_rights_de': drupal_record_set_de['attributes']['field_rights'],
Jonas Waeber's avatar
Jonas Waeber committed
174
175
                'field_rights_fr': drupal_record_set_fr['attributes']['field_rights'],
                'field_rights_it': drupal_record_set_it['attributes']['field_rights'],
176
                'field_scope_de': drupal_record_set_de['attributes']['field_scope'],
Jonas Waeber's avatar
Jonas Waeber committed
177
178
                'field_scope_fr': drupal_record_set_fr['attributes']['field_scope'],
                'field_scope_it': drupal_record_set_it['attributes']['field_scope'],
179
                'field_selection_de': drupal_record_set_de['attributes']['field_selection'],
Jonas Waeber's avatar
Jonas Waeber committed
180
181
182
                'field_selection_fr': drupal_record_set_fr['attributes']['field_selection'],
                'field_selection_it': drupal_record_set_it['attributes']['field_selection'],
                'field_supported_by_memoriav':
183
184
185
                    drupal_record_set_de['attributes']['field_supported_by_memoriav'],
                'field_time_period': drupal_record_set_de['attributes']['field_time_period'],
                'field_transfer_date': drupal_record_set_de['attributes']['field_transfer_date'],
Jonas Waeber's avatar
Jonas Waeber committed
186
                'field_image_gallery':
187
                    drupal_record_set_de['relationships']['field_image_gallery'],
Jonas Waeber's avatar
Jonas Waeber committed
188
                'field_metadata_languages': metadata_language_codes,
189
190
191
192
                'field_resp_institution_access': access_institution_ids,
                'field_resp_institution_master': master_institutions_ids,
                'field_resp_institution_original': original_institution_ids,
                'field_teaser_image': drupal_record_set_de['relationships']['field_teaser_image']
Jonas Waeber's avatar
Jonas Waeber committed
193
194
195
196
            }
        except LookupError as ex:
            msg = 'LookupError for ' + record_set_id + ': ' + str(ex) + '\n' + \
                  traceback.format_exc() + '\n' + \
197
                  'baseRequest: ' + de_drupal_url + '\n'
Jonas Waeber's avatar
Jonas Waeber committed
198
199
200
            current_app.logger.error(msg)
            return {
                       'status': 'FAILURE',
Jonas Waeber's avatar
Jonas Waeber committed
201
202
                       'topic_key': result_topic_value.get('field_memobase_id'),
                       'result_topic_value': result_topic_value
Jonas Waeber's avatar
Jonas Waeber committed
203
204
205
206
                   }, 500
        except Exception as ex:
            msg = 'Exception for ' + record_set_id + ': ' + str(ex) + '\n' + \
                  traceback.format_exc() + '\n' + \
207
                  'baseRequest: ' + de_drupal_url + '\n'
Jonas Waeber's avatar
Jonas Waeber committed
208
209
210
            current_app.logger.error(msg)
            return {
                       'status': 'FAILURE',
Jonas Waeber's avatar
Jonas Waeber committed
211
212
                       'topic_key': result_topic_value.get('field_memobase_id'),
                       'result_topic_value': result_topic_value,
Jonas Waeber's avatar
Jonas Waeber committed
213
214
215
216
217
218
219
                   }, 500

        # 2. write info into kafka topic
        try:
            producer = KafkaProducer(bootstrap_servers=current_app.config['kafka-broker-url'],
                                     value_serializer=lambda m: json.dumps(m, ensure_ascii=False)
                                     .encode('utf-8'))
Jonas Waeber's avatar
Jonas Waeber committed
220
221
            key = bytes(result_topic_value.get('field_memobase_id'), encoding='utf-8')
            producer.send(current_app.config['topic-drupal-export'], result_topic_value, key=key)
Jonas Waeber's avatar
Jonas Waeber committed
222
223
224
225
226
227
        except Exception as ex:
            msg = 'Exception for ' + record_set_id + ': ' + str(ex) + '\n' + \
                  traceback.format_exc()
            current_app.logger.error(msg)
            return {
                       'status': 'FAILURE',
Jonas Waeber's avatar
Jonas Waeber committed
228
229
                       'topic_key': result_topic_value.get('field_memobase_id'),
                       'result_topic_value': result_topic_value,
Jonas Waeber's avatar
Jonas Waeber committed
230
231
232
233
234
235
                       'exception': msg
                   }, 500

        current_app.logger.debug('success for ' + record_set_id)
        return {
                   'status': 'SUCCESS',
Jonas Waeber's avatar
Jonas Waeber committed
236
237
                   'topic_key': result_topic_value.get('field_memobase_id'),
                   'result_topic_value': result_topic_value
Jonas Waeber's avatar
Jonas Waeber committed
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
               }, 200

    @staticmethod
    def get_institution_id_list(institution_data, headers, auth):
        institution_ids = []
        for institution in institution_data:
            drupal_institution = \
                requests.get(current_app.config['drupal-api-url'] + '/jsonapi/node/' +
                             'institution/' + institution['id'], headers=headers, auth=auth)
            institution_ids.append(
                drupal_institution.json()['data']['attributes']['field_memobase_id']
            )
        return institution_ids

    @staticmethod
    def get_related_record_sets(fields):
        related_record_sets = fields['attributes']['field_related_record_sets']
        revised_related_record_sets = []
        for relatedRecordSet in related_record_sets:
            if 'entity:node' in relatedRecordSet['uri']:
                relatedRecordSet['uri'] = fields['attributes']['field_memobase_id']
                relatedRecordSet['title'] = 'internal'
            revised_related_record_sets.append(relatedRecordSet)
        return revised_related_record_sets