Skip to content

GitLab

Commit 337b76d7 authored by Jonas Waeber's avatar Jonas Waeber
Browse files

Remove old file for record set endpoint again.

parent 40904fc4
Pipeline #26148 passed with stages
in 2 minutes and 35 seconds
Hide whitespace changes
Inline Side-by-side
import_api_app/resources/MonitorDrupalJsonApiRecordSet.py deleted 100644 → 0
View file @ 40904fc4
from flask_restful import Resource, current_app
from kafka import KafkaProducer
import requests
import json
import traceback
from requests.auth import HTTPBasicAuth
class MonitorDrupalJsonApiRecordSet(Resource):
# Todo write/correct comment for swagger
def get(self, recordSetId):
"""
Get uuid of updated recordset form drupal, gets more information form \
drupal and puts it into the dedicated kafka topic
---
tags:
- Monitor Drupal JSON API
parameters:
- in: path
name: recordSetId
required: true
description: The UUID of the updated recordset
example: 0c4c777c-94f8-45ba-945a-bfe6967d40da
type: string
responses:
200:
description: Success, the information has been written into
the kafka topic
schema:
properties:
status:
type: string
example: SUCCESS/FAILURE
enum: ['SUCCESS', 'FAILURE']
topic_value:
type: string/json
example: the value written into the topic
"""
topicValue = ''
# 1. request info from drupal with received uuid
baseRequest = current_app.config['drupal-api-url'] + '/jsonapi/node/' + \
'record_set/' + recordSetId
baseResponse = None
user = current_app.config['drupal-user']
password = current_app.config['drupal-password']
auth = HTTPBasicAuth(user, password)
headers = {'X-API-Key': current_app.config['drupal-api-key']}
try:
baseResponse = requests.get(baseRequest, headers=headers, auth=auth)
drupalRecordSet = baseResponse.json()['data']
drupalRecordSet_fr =\
requests.get(current_app.config['drupal-api-url'] + '/fr/jsonapi/node/' +
'record_set/' + recordSetId, headers=headers, auth=auth).json()['data']
drupalRecordSet_it =\
requests.get(current_app.config['drupal-api-url'] + '/it/jsonapi/node/' +
'record_set/' + recordSetId, headers=headers, auth=auth).json()['data']
institutions = drupalRecordSet['relationships']['field_institution']['data']
institutionIds = self.getInstitutionIdList(institutions, headers, auth)
institutions = drupalRecordSet['relationships']['field_resp_institution_access']['data']
field_resp_institution_access = self.getInstitutionIdList(institutions, headers, auth)
institutions = drupalRecordSet['relationships']['field_resp_institution_master']['data']
field_resp_institution_master = self.getInstitutionIdList(institutions, headers, auth)
institutions =\
drupalRecordSet['relationships']['field_resp_institution_original']['data']
field_resp_institution_original = self.getInstitutionIdList(institutions, headers, auth)
metadataLanguageCodes = []
metadataLanguages =\
drupalRecordSet['relationships']['field_metadata_languages']['data']
for metadataLanguage in metadataLanguages:
drupalRecordSetLanguageCode = \
requests.get(current_app.config['drupal-api-url'] + '/jsonapi/taxonomy_term/' +
'language_of_metadata/' + metadataLanguage['id'],
headers=headers, auth=auth)
metadataLanguageCodes.append(
drupalRecordSetLanguageCode.json()['data']['attributes']['name']
)
field_related_record_sets = self.getRelatedRecordSets(drupalRecordSet)
field_related_record_sets_fr = self.getRelatedRecordSets(drupalRecordSet_fr)
field_related_record_sets_it = self.getRelatedRecordSets(drupalRecordSet_it)
topicValue = {
'type': drupalRecordSet['type'],
'status': drupalRecordSet['attributes']['status'],
'title_de': drupalRecordSet['attributes']['title'],
'title_fr': drupalRecordSet_fr['attributes']['title'],
'title_it': drupalRecordSet_it['attributes']['title'],
'field_institution': institutionIds,
'field_metadata_language_codes': metadataLanguageCodes,
'computed_teaser_image_url':
drupalRecordSet['attributes']['computed_teaser_image_url'],
'field_processed_teaser_text_de':
drupalRecordSet['attributes']['field_processed_teaser_text'],
'field_processed_teaser_text_fr':
drupalRecordSet_fr['attributes']['field_processed_teaser_text'],
'field_processed_teaser_text_it':
drupalRecordSet_it['attributes']['field_processed_teaser_text'],
'field_old_memobase_id':
drupalRecordSet['attributes']['field_old_memobase_id'],
'field_access_de': drupalRecordSet['attributes']['field_access'],
'field_access_fr': drupalRecordSet_fr['attributes']['field_access'],
'field_access_it': drupalRecordSet_it['attributes']['field_access'],
'field_access_memobase_de':
drupalRecordSet['attributes']['field_access_memobase'],
'field_access_memobase_fr':
drupalRecordSet_fr['attributes']['field_access_memobase'],
'field_access_memobase_it':
drupalRecordSet_it['attributes']['field_access_memobase'],
'field_content_de': drupalRecordSet['attributes']['field_content'],
'field_content_fr': drupalRecordSet_fr['attributes']['field_content'],
'field_content_it': drupalRecordSet_it['attributes']['field_content'],
'field_context_de': drupalRecordSet['attributes']['field_context'],
'field_context_fr': drupalRecordSet_fr['attributes']['field_context'],
'field_context_it': drupalRecordSet_it['attributes']['field_context'],
'field_data_transfer_de': drupalRecordSet['attributes']['field_data_transfer'],
'field_data_transfer_fr': drupalRecordSet_fr['attributes']['field_data_transfer'],
'field_data_transfer_it': drupalRecordSet_it['attributes']['field_data_transfer'],
'field_documents_de': drupalRecordSet['attributes']['field_documents'],
'field_documents_fr': drupalRecordSet_fr['attributes']['field_documents'],
'field_documents_it': drupalRecordSet_it['attributes']['field_documents'],
'field_info_on_development_de':
drupalRecordSet['attributes']['field_info_on_development'],
'field_info_on_development_fr':
drupalRecordSet_fr['attributes']['field_info_on_development'],
'field_info_on_development_it':
drupalRecordSet_it['attributes']['field_info_on_development'],
'field_language_de': drupalRecordSet['attributes']['field_language'],
'field_language_fr': drupalRecordSet_fr['attributes']['field_language'],
'field_language_it': drupalRecordSet_it['attributes']['field_language'],
'field_memobase_id': drupalRecordSet['attributes']['field_memobase_id'],
'field_notes': drupalRecordSet['attributes']['field_notes'],
'field_original_description_de':
drupalRecordSet['attributes']['field_original_description'],
'field_original_description_fr':
drupalRecordSet_fr['attributes']['field_original_description'],
'field_original_description_it':
drupalRecordSet_it['attributes']['field_original_description'],
'field_original_id': drupalRecordSet['attributes']['field_original_id'],
'field_original_shelf_mark':
drupalRecordSet['attributes']['field_original_shelf_mark'],
'field_original_title_de': drupalRecordSet['attributes']['field_original_title'],
'field_original_title_fr':
drupalRecordSet_fr['attributes']['field_original_title'],
'field_original_title_it':
drupalRecordSet_it['attributes']['field_original_title'],
'field_project_de': drupalRecordSet['attributes']['field_project'],
'field_project_fr': drupalRecordSet_fr['attributes']['field_project'],
'field_project_it': drupalRecordSet_it['attributes']['field_project'],
'field_publications_de': drupalRecordSet['attributes']['field_publications'],
'field_publications_fr': drupalRecordSet_fr['attributes']['field_publications'],
'field_publications_it': drupalRecordSet_it['attributes']['field_publications'],
'field_related_record_sets_de': field_related_record_sets,
'field_related_record_sets_fr': field_related_record_sets_fr,
'field_related_record_sets_it': field_related_record_sets_it,
'field_rights_de': drupalRecordSet['attributes']['field_rights'],
'field_rights_fr': drupalRecordSet_fr['attributes']['field_rights'],
'field_rights_it': drupalRecordSet_it['attributes']['field_rights'],
'field_scope_de': drupalRecordSet['attributes']['field_scope'],
'field_scope_fr': drupalRecordSet_fr['attributes']['field_scope'],
'field_scope_it': drupalRecordSet_it['attributes']['field_scope'],
'field_selection_de': drupalRecordSet['attributes']['field_selection'],
'field_selection_fr': drupalRecordSet_fr['attributes']['field_selection'],
'field_selection_it': drupalRecordSet_it['attributes']['field_selection'],
'field_supported_by_memoriav':
drupalRecordSet['attributes']['field_supported_by_memoriav'],
'field_time_period': drupalRecordSet['attributes']['field_time_period'],
'field_transfer_date': drupalRecordSet['attributes']['field_transfer_date'],
'field_image_gallery':
drupalRecordSet['relationships']['field_image_gallery'],
'field_metadata_languages': metadataLanguageCodes,
'field_resp_institution_access': field_resp_institution_access,
'field_resp_institution_master': field_resp_institution_master,
'field_resp_institution_original': field_resp_institution_original,
'field_teaser_image': drupalRecordSet['relationships']['field_teaser_image']
}
except LookupError as ex:
msg = 'LookupError for ' + recordSetId + ': ' + str(ex) + '\n' +\
traceback.format_exc() + '\n' + \
'baseRequest: ' + baseRequest + '\n' + \
'baseResponse: ' + baseResponse.content.decode("utf-8")
current_app.logger.error(msg)
return {
'status': 'FAILURE',
'topic_key': topicValue.get('field_memobase_id'),
'topic_value': topicValue
}, 500
except Exception as ex:
msg = 'Exception for ' + recordSetId + ': ' + str(ex) + '\n' + \
traceback.format_exc() + '\n' + \
'baseRequest: ' + baseRequest + '\n' + \
'baseResponse: ' + baseResponse.content.decode("utf-8")
current_app.logger.error(msg)
return {
'status': 'FAILURE',
'topic_key': topicValue.get('field_memobase_id'),
'topic_value': topicValue,
}, 500
# 2. write info into kafka topic
try:
producer = KafkaProducer(bootstrap_servers=current_app.config['kafka-broker-url'],
value_serializer=lambda m: json.dumps(m, ensure_ascii=False)
.encode('utf-8'))
key = bytes(topicValue.get('field_memobase_id'), encoding='utf-8')
producer.send(current_app.config['topic-drupal-export'], topicValue, key=key)
except Exception as ex:
msg = 'Exception for ' + recordSetId + ': ' + str(ex) + '\n' +\
traceback.format_exc()
current_app.logger.error(msg)
return {
'status': 'FAILURE',
'topic_key': topicValue.get('field_memobase_id'),
'topic_value': topicValue,
'exception': msg
}, 500
current_app.logger.debug('success for ' + recordSetId)
return {
'status': 'SUCCESS',
'topic_key': topicValue.get('field_memobase_id'),
'topic_value': topicValue
}, 200
def getInstitutionIdList(self, institutionData, headers, auth):
institutionIds = []
for institution in institutionData:
institution
drupalInstitution = \
requests.get(current_app.config['drupal-api-url'] + '/jsonapi/node/' +
'institution/' + institution['id'], headers=headers, auth=auth)
institutionIds.append(
drupalInstitution.json()['data']['attributes']['field_memobase_id']
)
return institutionIds
def getRelatedRecordSets(self, fields):
relatedRecordSets = fields['attributes']['field_related_record_sets']
revisedRelatedRecordSets = []
for relatedRecordSet in relatedRecordSets:
if 'entity:node' in relatedRecordSet['uri']:
relatedRecordSet['uri'] = fields['attributes']['field_memobase_id']
relatedRecordSet['title'] = 'internal'
revisedRelatedRecordSets.append(relatedRecordSet)
return revisedRelatedRecordSets
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment