Unverified Commit e75c3883 authored by Sebastian Schüpbach's avatar Sebastian Schüpbach
Browse files

show more info in report when errors occur

parent fcf8f2bd
Pipeline #19069 passed with stages
in 2 minutes and 1 second
......@@ -177,7 +177,8 @@ def _is_directly_fetchable(digital_object_resource) -> bool:
def _get_access_status(graph, record_id) -> str:
"""
Decide on access status. Possible values are `public`, `closed`, and `unavailable`
Decide on access status. Possible values are `public`, `closed`, `unavailable`,
`faro`, `onsite`, or `noonsite`
"""
for resource in graph:
if 'type' in resource and resource['type'] == 'access' and \
......@@ -193,7 +194,7 @@ def _get_access_status(graph, record_id) -> str:
else:
logging.info(f'Digital object of record {record_id} has access type ' +
f'`{resource["name"]}`. This makes the media resource unavailable.')
return 'unavailable'
return resource['name']
logging.info(f'Digital object of record {record_id} has no related access information!'
+ ' Media resource is therefore unavailable')
return 'unavailable'
......@@ -287,20 +288,27 @@ class MediametadataToDB:
record_processor.new_record(record_id)
access_status = _get_access_status(record, record_id)
for record_resource in record:
if _is_digital_object(record_resource) and _is_playable(access_status):
enriched_data = _extract_fields(record_resource,
_extract_digital_object_values,
access_status)
if enriched_data:
record_processor.digital_object_ok(record_id, enriched_data)
if _has_audio_snippet(enriched_data):
snippet_entry = _create_audio_snippet_entry(enriched_data)
if snippet_entry:
record_processor.audio_snippet_ok(record_id, snippet_entry)
else:
record_processor.audio_snippet_fail(record_id)
if _is_digital_object(record_resource):
if _is_playable(access_status):
enriched_data = _extract_fields(record_resource,
_extract_digital_object_values,
access_status)
if enriched_data:
record_processor.digital_object_ok(record_id, enriched_data)
if _has_audio_snippet(enriched_data):
snippet_entry = _create_audio_snippet_entry(enriched_data)
if snippet_entry:
record_processor.audio_snippet_ok(record_id,
snippet_entry)
else:
record_processor.audio_snippet_fail(record_id)
else:
record_processor.digital_object_fail(record_id)
else:
record_processor.digital_object_fail(record_id)
record_processor.digital_object_ignore(record_id,
f"Ignored because of"
f" access status "
f"{access_status}")
elif _is_thumbnail(record_resource):
enriched_data = _extract_fields(record_resource,
_extract_thumbnail_values,
......
......@@ -56,7 +56,7 @@ class Indexer:
return 'INSERT INTO {} ({}) VALUES ({}) ON DUPLICATE KEY UPDATE {}'.format(
table_name, db_fields, db_values, key_value)
def insert_in_db(self, record) -> bool:
def insert_in_db(self, record) -> (bool, str):
"""
Insert record in DB
"""
......@@ -68,10 +68,10 @@ class Indexer:
try:
self.mariadb_cursor.execute(entities_stmt)
self.mariadb_cursor.execute(metadata_stmt)
return True
except Exception:
return True, ""
except mariadb.Error as ex:
logging.error("Problems in sql statement: {}".format(entities_stmt))
return False
return False, str(ex)
def commit(self):
"""
......
......@@ -25,6 +25,7 @@ class RecordProcessor:
self.processed_records[rec_id]['digital_object'] = \
{'data': data,
'ok': True,
'ignored': False,
'msg': 'successful'}
def digital_object_fail(self, rec_id):
......@@ -32,14 +33,24 @@ class RecordProcessor:
self.processed_records[rec_id]['digital_object'] = \
{'data': None,
'ok': False,
'ignored': False,
'msg': 'parsing failed'}
def digital_object_ignore(self, rec_id, message):
logging.info(f"Digital object resource for {rec_id} ignored")
self.processed_records[rec_id]['digital_object'] = \
{'data': None,
'ok': True,
'ignored': True,
'msg': message}
def thumbnail_ok(self, rec_id, data):
logging.debug(f"Parsing of thumbnail resource for {rec_id} successful")
self.counter += 1
self.processed_records[rec_id]['thumbnail'] = \
{'data': data,
'ok': True,
'ignored': False,
'msg': 'successful'}
def thumbnail_fail(self, rec_id):
......@@ -47,6 +58,7 @@ class RecordProcessor:
self.processed_records[rec_id]['thumbnail'] = \
{'data': None,
'ok': False,
'ignored': False,
'msg': 'parsing failed'}
def audio_snippet_ok(self, rec_id, data):
......@@ -55,6 +67,7 @@ class RecordProcessor:
self.processed_records[rec_id]['audio_snippet'] = \
{'data': data,
'ok': True,
'ignored': False,
'msg': 'successful'
}
......@@ -63,6 +76,7 @@ class RecordProcessor:
self.processed_records[rec_id]['audio_snippet'] = \
{'data': None,
'ok': False,
'ignored': False,
'msg': 'parsing failed'}
def abort(self, ex):
......@@ -74,11 +88,11 @@ class RecordProcessor:
for key in self.processed_records.keys():
record = self.processed_records[key]
dig_obj_msg = \
record['digital_object']['msg'] if 'digital_object' in record else 'ignored'
record['digital_object']['msg'] if 'digital_object' in record else 'not available'
thumbnail_msg = \
record['thumbnail']['msg'] if 'thumbnail' in record else 'ignored'
record['thumbnail']['msg'] if 'thumbnail' in record else 'not available'
audio_snip_msg = \
record['audio_snippet']['msg'] if 'audio_snippet' in record else 'ignored'
record['audio_snippet']['msg'] if 'audio_snippet' in record else 'not available'
if RecordProcessor._parsing_errors(record):
self.reporter.send_message(key, 'FATAL',
......@@ -87,15 +101,16 @@ class RecordProcessor:
.format(dig_obj_msg, thumbnail_msg, audio_snip_msg))
else:
ok = True
if 'digital_object' in record:
err_msg = ""
if 'digital_object' in record and not record['digital_object']['ignored']:
logging.debug(f"Indexing digital object for {key} in DB")
ok = self.indexer.insert_in_db(record['digital_object']['data'])
if ok and 'thumbnail' in record:
ok, err_msg = self.indexer.insert_in_db(record['digital_object']['data'])
if ok and 'thumbnail' in record and not record['thumbnail']['ignored']:
logging.debug(f"Indexing thumbnail for {key} in DB")
ok = self.indexer.insert_in_db(record['thumbnail']['data'])
if ok and 'audio_snippet' in record:
ok, err_msg = self.indexer.insert_in_db(record['thumbnail']['data'])
if ok and 'audio_snippet' in record and not record['audio_snippet']['ignored']:
logging.debug(f"Indexing audio snippet for {key} in DB")
ok = self.indexer.insert_in_db(record['audio_snippet']['data'])
ok, err_msg = self.indexer.insert_in_db(record['audio_snippet']['data'])
if ok:
self.indexer.commit()
self.reporter.send_message(key, 'SUCCESS',
......@@ -106,5 +121,5 @@ class RecordProcessor:
else:
self.indexer.rollback()
self.reporter.send_message(key, 'FATAL',
'Indexing failed')
f'Indexing failed: {err_msg}')
self.processed_records.clear()
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment