Unverified Commit facbc17d authored by Sebastian Schüpbach's avatar Sebastian Schüpbach
Browse files

report preciser error message when parsing fails

parent 8556a098
Pipeline #19808 failed with stages
in 38 seconds
......@@ -62,18 +62,19 @@ def _connect_to_kafka(retries=0):
def _extract_fields(record_json_data,
fetch_from_obj_fun,
access_status) -> dict:
access_status) -> (dict, str):
"""
Extract fields from JSON object by applying `fetch_from_obj_fun` function
"""
if 'locator' in record_json_data and '@id' in record_json_data:
return fetch_from_obj_fun(record_json_data, access_status)
return fetch_from_obj_fun(record_json_data, access_status), None
elif '@id' in record_json_data:
logging.info('Record ' + record_json_data['@id'] +
' does not have a locator-property.')
' does not have a locator property.')
return dict(), 'No locator property found'
else:
logging.warning('Record without @id-property detected!')
return dict()
return dict(), 'No @id property found'
def _extract_thumbnail_values(msg, _access_status) -> dict:
......@@ -290,7 +291,7 @@ class MediametadataToDB:
for record_resource in record:
if _is_digital_object(record_resource):
if _is_playable(access_status):
enriched_data = _extract_fields(record_resource,
enriched_data, error = _extract_fields(record_resource,
_extract_digital_object_values,
access_status)
if enriched_data:
......@@ -303,20 +304,20 @@ class MediametadataToDB:
else:
record_processor.audio_snippet_fail(record_id)
else:
record_processor.digital_object_fail(record_id)
record_processor.digital_object_fail(record_id, error)
else:
record_processor.digital_object_ignore(record_id,
f"Ignored because of"
f" access status "
f"{access_status}")
elif _is_thumbnail(record_resource):
enriched_data = _extract_fields(record_resource,
enriched_data, error = _extract_fields(record_resource,
_extract_thumbnail_values,
access_status)
if enriched_data:
record_processor.thumbnail_ok(record_id, enriched_data)
else:
record_processor.thumbnail_fail(record_id)
record_processor.thumbnail_fail(record_id, error)
ok = record_processor.index()
if ok:
consumer.commit()
......
......@@ -29,13 +29,13 @@ class RecordProcessor:
'ignored': False,
'msg': 'successful'}
def digital_object_fail(self, rec_id):
def digital_object_fail(self, rec_id, err):
logging.warning(f"Parsing of digital object resource for {rec_id} failed")
self.processed_records[rec_id]['digital_object'] = \
{'data': None,
'ok': False,
'ignored': False,
'msg': 'parsing failed'}
'msg': 'parsing failed' + f': {err}' if err else ''}
def digital_object_ignore(self, rec_id, message):
logging.info(f"Digital object resource for {rec_id} ignored")
......@@ -54,13 +54,13 @@ class RecordProcessor:
'ignored': False,
'msg': 'successful'}
def thumbnail_fail(self, rec_id):
def thumbnail_fail(self, rec_id, err):
logging.warning(f"Parsing of thumbnail resource for {rec_id} failed")
self.processed_records[rec_id]['thumbnail'] = \
{'data': None,
'ok': False,
'ignored': False,
'msg': 'parsing failed'}
'msg': 'parsing failed' + f': {err}' if err else ''}
def audio_snippet_ok(self, rec_id, data):
logging.debug(f"Parsing of audio snippet resource for {rec_id} successful")
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment