Unverified Commit 052eb2ca authored by Sebastian Schüpbach's avatar Sebastian Schüpbach
Browse files

write to db after 100 processed records

parent 6c72e37d
Pipeline #16988 canceled with stages
...@@ -213,7 +213,6 @@ def _create_entities_entry(record, mariadb_cursor): ...@@ -213,7 +213,6 @@ def _create_entities_entry(record, mariadb_cursor):
def _create_metadata_entry(record, mariadb_cursor): def _create_metadata_entry(record, mariadb_cursor):
fields = ['sig', 'mimetype', 'height', 'width', 'duration', 'type'] fields = ['sig', 'mimetype', 'height', 'width', 'duration', 'type']
sql_stmt = _create_sql_stmt('metadata', record, fields) sql_stmt = _create_sql_stmt('metadata', record, fields)
logging.info(sql_stmt)
mariadb_cursor.execute(sql_stmt) mariadb_cursor.execute(sql_stmt)
...@@ -277,7 +276,7 @@ class MediametadataToDB: ...@@ -277,7 +276,7 @@ class MediametadataToDB:
try: try:
while True: while True:
consumer.poll(max_records=100) consumer.poll(max_records=100)
for recordsJson in consumer: for recordNo, recordsJson in enumerate(consumer):
records_json_data = recordsJson.value['@graph'] records_json_data = recordsJson.value['@graph']
record_id = _get_record_id(records_json_data) record_id = _get_record_id(records_json_data)
logging.debug(f'Processing record {record_id}') logging.debug(f'Processing record {record_id}')
...@@ -301,6 +300,14 @@ class MediametadataToDB: ...@@ -301,6 +300,14 @@ class MediametadataToDB:
"SUCCESS", "SUCCESS",
"Ignoring record since access of " + "Ignoring record since access of " +
"digitalObject is unavailable") "digitalObject is unavailable")
if recordNo % 100 == 0:
_write_values_in_db(mariadb_cursor, record_values_for_db)
mariadb_connection.commit()
for record_value in record_values_for_db:
logging.info(f'Record {record_value["sig"]} successfully indexed')
reporter.send_message(record_value['sig'], "SUCCESS", "Indexing successful")
record_values_for_db = []
consumer.commit()
# arriving here means there are no new messages to poll from # arriving here means there are no new messages to poll from
_write_values_in_db(mariadb_cursor, record_values_for_db) _write_values_in_db(mariadb_cursor, record_values_for_db)
mariadb_connection.commit() mariadb_connection.commit()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment