Unverified Commit 045408f3 authored by Sebastian Schüpbach's avatar Sebastian Schüpbach
Browse files

add read records counter


Signed-off-by: Sebastian Schüpbach's avatarSebastian Schüpbach <sebastian.schuepbach@unibas.ch>
parent 771f7c16
Pipeline #17931 passed with stages
in 1 minute and 56 seconds
......@@ -279,10 +279,12 @@ class MediametadataToDB:
# process messages:
record_values_for_db = []
reporter = Reporter()
counter = 0
try:
while True:
consumer.poll()
for recordNo, recordsJson in enumerate(consumer):
counter += 1
records_json_data = recordsJson.value['@graph']
record_id = _get_record_id(records_json_data)
logging.debug(f'Processing record {record_id}')
......@@ -316,6 +318,8 @@ class MediametadataToDB:
"Indexing successful")
record_values_for_db = []
consumer.commit()
if counter % 1000 == 0:
logging.info('{} messages read till now'.format(counter))
# arriving here means there are no new messages to poll from
_write_values_in_db(mariadb_cursor, record_values_for_db)
mariadb_connection.commit()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment