Unverified Commit f93d74f8 authored by Sebastian Schüpbach's avatar Sebastian Schüpbach
Browse files

write report in every case

parent 045408f3
Pipeline #17932 passed with stages
in 1 minute and 52 seconds
......@@ -79,30 +79,22 @@ def _connect_to_mariadb(retries=0):
def _try_fetch_from_json_object(record_json_data,
record_values_for_db,
fetch_from_obj_fun,
access_status):
if 'locator' in record_json_data:
values = fetch_from_obj_fun(record_json_data, access_status)
record_values_for_db.append(values)
if 'locator' in record_json_data and '@id' in record_json_data:
return fetch_from_obj_fun(record_json_data, access_status)
elif '@id' in record_json_data:
logging.info('Record ' + record_json_data['@id'] +
' does not have a locator-property.')
else:
logging.warning('Record without @id-property detected!')
return None
def _get_values_from_thumbnail_object(msg, _access_status):
return_values = {
'mimetype': 'image/jp2',
'type': 'image',
'access': 'public',
'proto': 'file'
}
if '@id' in msg:
return_values['sig'] = '{}-poster'.format(msg['@id'].split('/')[-2])
if 'locator' in msg:
return_values['uri'] = 'file:///data/{}-poster.jp2'.format(msg['@id'].split('/')[-2])
return_values = {'mimetype': 'image/jp2', 'type': 'image', 'access': 'public', 'proto': 'file',
'sig': '{}-poster'.format(msg['@id'].split('/')[-2]),
'uri': 'file:///data/{}-poster.jp2'.format(msg['@id'].split('/')[-2])}
if 'height' in msg:
height = msg['height']
return_values['height'] = height
......@@ -113,12 +105,11 @@ def _get_values_from_thumbnail_object(msg, _access_status):
def _get_values_from_digital_object(msg, access_status):
if 'isDistributedOn' not in msg:
logging.warning("No isDistributedOn property found in object")
return None
file_extension = ''
return_values = {
'access': access_status
}
if '@id' in msg:
return_values['sig'] = msg['@id'].split('/')[-1]
return_values = {'access': access_status, 'sig': msg['@id'].split('/')[-1]}
if 'height' in msg:
height = msg['height']
return_values['height'] = height
......@@ -128,8 +119,7 @@ def _get_values_from_digital_object(msg, access_status):
if 'duration' in msg:
duration = msg['duration']
return_values['duration'] = duration
if 'isDistributedOn' in msg:
return_values['type'] = msg['isDistributedOn']
return_values['type'] = msg['isDistributedOn']
if 'hasMimeType' in msg:
if return_values['type'] == 'image':
mimetype = 'image/jp2'
......@@ -224,6 +214,7 @@ def _has_audio_snippet(record):
record['uri'].startswith('file://')
# TODO: Eventually remove
def _has_http_locator(digital_object):
'locator' in digital_object and digital_object['locator'].startswith('http')
......@@ -291,17 +282,38 @@ class MediametadataToDB:
access_status = _get_access_status(records_json_data, record_id)
if access_status == 'public' or access_status == 'closed':
for recordJsonData in records_json_data:
enrichable = False
if 'type' in recordJsonData and \
recordJsonData['type'] == 'digitalObject' and \
_has_http_locator(recordJsonData):
_try_fetch_from_json_object(recordJsonData, record_values_for_db,
_get_values_from_digital_object,
access_status)
enrichable = True
enriched_data = \
_try_fetch_from_json_object(recordJsonData,
_get_values_from_digital_object,
access_status)
if not enriched_data:
reporter.send_message(record_id,
"FATAL",
"Could not process digitalObject")
else:
record_values_for_db.append(enriched_data)
if 'type' in recordJsonData and \
recordJsonData['type'] == 'thumbnail':
_try_fetch_from_json_object(recordJsonData, record_values_for_db,
_get_values_from_thumbnail_object,
access_status)
enrichable = True
enriched_data = \
_try_fetch_from_json_object(recordJsonData,
_get_values_from_thumbnail_object,
access_status)
if not enriched_data:
reporter.send_message(record_id,
"FATAL",
"Could not process thumbnail object")
else:
record_values_for_db.append(enriched_data)
if not enrichable:
reporter.send_message(record_id,
"IGNORE",
"Resource has no digitalObject or thumbnail")
else:
logging.info(f'Ignoring record {record_id} since' +
f' access of digitalObject is {access_status}')
......
......@@ -130,10 +130,11 @@ class Test(TestCase):
Test._load_file_and_get_res('SRF-UMusik-BE_MG_DUR_B62276.json', lambda x: x):
if 'type' in recordJsonData and \
recordJsonData['type'] == 'digitalObject':
MediametadataToDB._try_fetch_from_json_object(recordJsonData, record_values_for_db,
MediametadataToDB.
_get_values_from_digital_object,
'public')
record_values_for_db.append(
MediametadataToDB._try_fetch_from_json_object(recordJsonData,
MediametadataToDB.
_get_values_from_digital_object,
'public'))
self.assertFalse([MediametadataToDB._has_audio_snippet(record)
for record in record_values_for_db][0])
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment