Unverified Commit b3879d13 authored by Sebastian Schüpbach's avatar Sebastian Schüpbach
Browse files

make logging level dynamic


Signed-off-by: Sebastian Schüpbach's avatarSebastian Schüpbach <sebastian.schuepbach@unibas.ch>
parent a330f007
......@@ -30,6 +30,8 @@ spec:
value: "fedora-output-json-records"
- name: REPORTING_TOPIC
value: "postprocessing-reporting"
- name: LOG_LEVEL
value: "INFO"
envFrom:
- configMapRef:
name: "prod-kafka-bootstrap-servers"
......
......@@ -39,13 +39,30 @@
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import logging
import os
from mediametadatatodb_app.resources.MediametadataToDB import MediametadataToDB
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
numeric_level = getattr(logging, os.getenv('LOG_LEVEL').upper(), None)
if not isinstance(numeric_level, int):
raise ValueError(f'Invalid log level: {os.getenv("LOG_LEVEL")}')
logging.basicConfig(level=numeric_level)
logging.info("Starting up")
runner = MediametadataToDB()
runner.run()
......
......@@ -52,6 +52,19 @@
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published
# by the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import json
import logging
......@@ -209,6 +222,12 @@ def _get_access_status(graph):
return 'unavailable'
def _get_record_id(graph):
for resource in graph:
if '@type' in resource and resource['@type'] == 'https://www.ica.org/standards/RiC/ontology#Record':
return resource['@id'] if '@id' in resource else None
def _create_sql_stmt(table_name, record, fields):
db_fields = [dbField for dbField in fields
if dbField in record and record[dbField] is not None]
......@@ -296,6 +315,8 @@ class MediametadataToDB:
consumer.poll(max_records=100)
for recordsJson in consumer:
records_json_data = recordsJson.value['@graph']
record_id = _get_record_id(records_json_data)
logging.debug(f'Processing record {record_id}')
access_status = _get_access_status(records_json_data)
if access_status == 'public' or access_status == 'closed':
for recordJsonData in records_json_data:
......@@ -310,22 +331,24 @@ class MediametadataToDB:
_get_values_from_thumbnail_object,
access_status)
else:
logging.info('Ignoring record since access of digitalObject is {}'
.format(access_status))
logging.info(f'Ignoring record {record_id} since access of digitalObject is {access_status}')
reporter.send_message(record_id,
"SUCCESS",
"Ignoring record since access of " +
"digitalObject is unavailable")
# arriving here means there are no new messages to poll from
_write_values_in_db(mariadb_cursor, record_values_for_db)
mariadb_connection.commit()
for record_value in record_values_for_db:
# TODO: Create more meaningful reports!
logging.info(f'Record {record_value["@id"]} successfully indexed')
reporter.send_message(record_value['@id'], "SUCCESS", "Indexing successful")
record_values_for_db = []
consumer.commit()
except KafkaError as ex:
status = 'It was not possible to consume the Kafka messages.' + '\n' + str(ex)
logging.error(status)
except Exception as ex:
status = 'It was not possible to consume the Kafka messages.' + '\n' + str(ex)
logging.error(status)
for record_value in record_values_for_db:
reporter.send_message(record_value['@id'], "FAILURE", f"Indexing failed: {ex}")
def __init__(self):
# TODO : maybe take that to a configuration (development vs pod running in
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment