Unverified Commit a56365c6 authored by Sebastian Schüpbach's avatar Sebastian Schüpbach
Browse files

more logging

parent 418274c3
Pipeline #19040 failed with stages
in 36 seconds
import mysql.connector as mariadb
import logging import logging
import numbers import numbers
import os import os
import time import time
import mysql.connector as mariadb
# noinspection SqlResolve,SqlNoDataSourceInspection # noinspection SqlResolve,SqlNoDataSourceInspection
class Indexer: class Indexer:
...@@ -18,6 +19,8 @@ class Indexer: ...@@ -18,6 +19,8 @@ class Indexer:
Connect to MariaDB. Abort after configured retries. Connect to MariaDB. Abort after configured retries.
""" """
try: try:
logging.debug(f"Connecting to DB {os.environ['MARIADB_DATABASE']} on "
f"{os.environ['MARIADB_HOST']}:{os.environ['MARIADB_PORT']}")
mariadb_connection = mariadb.connect(user=os.environ['MARIADB_USER'], mariadb_connection = mariadb.connect(user=os.environ['MARIADB_USER'],
password=os.environ['MARIADB_PASSWORD'], password=os.environ['MARIADB_PASSWORD'],
host=os.environ['MARIADB_HOST'], host=os.environ['MARIADB_HOST'],
...@@ -74,11 +77,13 @@ class Indexer: ...@@ -74,11 +77,13 @@ class Indexer:
""" """
Commit changes to DB Commit changes to DB
""" """
logging.debug("Commiting changes to DB")
self.mariadb_connection.commit() self.mariadb_connection.commit()
def rollback(self): def rollback(self):
""" """
Rollback changes Rollback changes
""" """
logging.info("Rollback changes")
self.mariadb_cursor.reset() self.mariadb_cursor.reset()
self.mariadb_connection.rollback() self.mariadb_connection.rollback()
from mediametadatatodb_app.resources.reporter import Reporter from mediametadatatodb_app.resources.reporter import Reporter
from mediametadatatodb_app.resources.indexer import Indexer from mediametadatatodb_app.resources.indexer import Indexer
import logging
class RecordProcessor: class RecordProcessor:
...@@ -19,6 +20,7 @@ class RecordProcessor: ...@@ -19,6 +20,7 @@ class RecordProcessor:
self.processed_records[rec_id] = dict() self.processed_records[rec_id] = dict()
def digital_object_ok(self, rec_id, data): def digital_object_ok(self, rec_id, data):
logging.debug(f"Parsing of digital object resource for {rec_id} successful")
self.counter += 1 self.counter += 1
self.processed_records[rec_id]['digital_object'] = \ self.processed_records[rec_id]['digital_object'] = \
{'data': data, {'data': data,
...@@ -26,12 +28,14 @@ class RecordProcessor: ...@@ -26,12 +28,14 @@ class RecordProcessor:
'msg': 'successful'} 'msg': 'successful'}
def digital_object_fail(self, rec_id): def digital_object_fail(self, rec_id):
logging.warning(f"Parsing of digital object resource for {rec_id} failed")
self.processed_records[rec_id]['digital_object'] = \ self.processed_records[rec_id]['digital_object'] = \
{'data': None, {'data': None,
'ok': False, 'ok': False,
'msg': 'parsing failed'} 'msg': 'parsing failed'}
def thumbnail_ok(self, rec_id, data): def thumbnail_ok(self, rec_id, data):
logging.debug(f"Parsing of thumbnail resource for {rec_id} successful")
self.counter += 1 self.counter += 1
self.processed_records[rec_id]['thumbnail'] = \ self.processed_records[rec_id]['thumbnail'] = \
{'data': data, {'data': data,
...@@ -39,12 +43,14 @@ class RecordProcessor: ...@@ -39,12 +43,14 @@ class RecordProcessor:
'msg': 'successful'} 'msg': 'successful'}
def thumbnail_fail(self, rec_id): def thumbnail_fail(self, rec_id):
logging.warning(f"Parsing of thumbnail resource for {rec_id} failed")
self.processed_records[rec_id]['thumbnail'] = \ self.processed_records[rec_id]['thumbnail'] = \
{'data': None, {'data': None,
'ok': False, 'ok': False,
'msg': 'parsing failed'} 'msg': 'parsing failed'}
def audio_snippet_ok(self, rec_id, data): def audio_snippet_ok(self, rec_id, data):
logging.debug(f"Parsing of audio snippet resource for {rec_id} successful")
self.counter += 1 self.counter += 1
self.processed_records[rec_id]['audio_snippet'] = \ self.processed_records[rec_id]['audio_snippet'] = \
{'data': data, {'data': data,
...@@ -53,12 +59,14 @@ class RecordProcessor: ...@@ -53,12 +59,14 @@ class RecordProcessor:
} }
def audio_snippet_fail(self, rec_id): def audio_snippet_fail(self, rec_id):
logging.warning(f"Parsing of audio snippet resource for {rec_id} failed")
self.processed_records[rec_id]['audio_snippet'] = \ self.processed_records[rec_id]['audio_snippet'] = \
{'data': None, {'data': None,
'ok': False, 'ok': False,
'msg': 'parsing failed'} 'msg': 'parsing failed'}
def abort(self, ex): def abort(self, ex):
logging.error(f"Indexing failed. Aborting...")
for key in self.processed_records.keys(): for key in self.processed_records.keys():
self.reporter.send_message(key, 'FATAL', f'Indexing failed: {ex}') self.reporter.send_message(key, 'FATAL', f'Indexing failed: {ex}')
...@@ -80,10 +88,13 @@ class RecordProcessor: ...@@ -80,10 +88,13 @@ class RecordProcessor:
else: else:
ok = True ok = True
if 'digital_object' in record: if 'digital_object' in record:
logging.debug(f"Indexing digital object for {key} in DB")
ok = self.indexer.insert_in_db(record['digital_object']['data']) ok = self.indexer.insert_in_db(record['digital_object']['data'])
if ok and 'thumbnail' in record: if ok and 'thumbnail' in record:
logging.debug(f"Indexing thumbnail for {key} in DB")
ok = self.indexer.insert_in_db(record['thumbnail']['data']) ok = self.indexer.insert_in_db(record['thumbnail']['data'])
if ok and 'audio_snippet' in record: if ok and 'audio_snippet' in record:
logging.debug(f"Indexing audio snippet for {key} in DB")
ok = self.indexer.insert_in_db(record['audio_snippet']['data']) ok = self.indexer.insert_in_db(record['audio_snippet']['data'])
if ok: if ok:
self.indexer.commit() self.indexer.commit()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment