Unverified Commit 052eb2ca authored by Sebastian Schüpbach's avatar Sebastian Schüpbach
Browse files

write to db after 100 processed records

parent 6c72e37d
Pipeline #16988 canceled with stages
......@@ -213,7 +213,6 @@ def _create_entities_entry(record, mariadb_cursor):
def _create_metadata_entry(record, mariadb_cursor):
fields = ['sig', 'mimetype', 'height', 'width', 'duration', 'type']
sql_stmt = _create_sql_stmt('metadata', record, fields)
logging.info(sql_stmt)
mariadb_cursor.execute(sql_stmt)
......@@ -277,7 +276,7 @@ class MediametadataToDB:
try:
while True:
consumer.poll(max_records=100)
for recordsJson in consumer:
for recordNo, recordsJson in enumerate(consumer):
records_json_data = recordsJson.value['@graph']
record_id = _get_record_id(records_json_data)
logging.debug(f'Processing record {record_id}')
......@@ -301,6 +300,14 @@ class MediametadataToDB:
"SUCCESS",
"Ignoring record since access of " +
"digitalObject is unavailable")
if recordNo % 100 == 0:
_write_values_in_db(mariadb_cursor, record_values_for_db)
mariadb_connection.commit()
for record_value in record_values_for_db:
logging.info(f'Record {record_value["sig"]} successfully indexed')
reporter.send_message(record_value['sig'], "SUCCESS", "Indexing successful")
record_values_for_db = []
consumer.commit()
# arriving here means there are no new messages to poll from
_write_values_in_db(mariadb_cursor, record_values_for_db)
mariadb_connection.commit()
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment