In order to mitigate against the brute force attacks against Gitlab accounts, we are moving to all edu-ID Logins. We would like to remind you to link your account with your edu-id. Login will be possible only by edu-ID after November 30, 2021. Here you can find the instructions for linking your account.

If you don't have a SWITCH edu-ID, you can create one with this guide here

kind regards

This Server has been upgraded to GitLab release 14.2.6

Unverified Commit 052eb2ca authored by Sebastian Schüpbach's avatar Sebastian Schüpbach
Browse files

write to db after 100 processed records

parent 6c72e37d
Pipeline #16988 canceled with stages
......@@ -213,7 +213,6 @@ def _create_entities_entry(record, mariadb_cursor):
def _create_metadata_entry(record, mariadb_cursor):
fields = ['sig', 'mimetype', 'height', 'width', 'duration', 'type']
sql_stmt = _create_sql_stmt('metadata', record, fields)
logging.info(sql_stmt)
mariadb_cursor.execute(sql_stmt)
......@@ -277,7 +276,7 @@ class MediametadataToDB:
try:
while True:
consumer.poll(max_records=100)
for recordsJson in consumer:
for recordNo, recordsJson in enumerate(consumer):
records_json_data = recordsJson.value['@graph']
record_id = _get_record_id(records_json_data)
logging.debug(f'Processing record {record_id}')
......@@ -301,6 +300,14 @@ class MediametadataToDB:
"SUCCESS",
"Ignoring record since access of " +
"digitalObject is unavailable")
if recordNo % 100 == 0:
_write_values_in_db(mariadb_cursor, record_values_for_db)
mariadb_connection.commit()
for record_value in record_values_for_db:
logging.info(f'Record {record_value["sig"]} successfully indexed')
reporter.send_message(record_value['sig'], "SUCCESS", "Indexing successful")
record_values_for_db = []
consumer.commit()
# arriving here means there are no new messages to poll from
_write_values_in_db(mariadb_cursor, record_values_for_db)
mariadb_connection.commit()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment