make logging level dynamic

Signed-off-by: Sebastian Schüpbach <sebastian.schuepbach@unibas.ch>

k8s-manifests/test/kubernetes-deployment.yml

@@ -30,6 +30,8 @@ spec:
                 value: "fedora-output-json-records"
               - name: REPORTING_TOPIC
                 value: "postprocessing-reporting"
+              - name: LOG_LEVEL
+                value: "INFO"
             envFrom:
               - configMapRef:
                   name: "prod-kafka-bootstrap-servers"

mediametadatatodb_app/main.py

@@ -39,13 +39,30 @@
 #
 #  You should have received a copy of the GNU Affero General Public License
 #  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU Affero General Public License as published
+#  by the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU Affero General Public License for more details.
+#
+#  You should have received a copy of the GNU Affero General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import logging
+import os
 
 from mediametadatatodb_app.resources.MediametadataToDB import MediametadataToDB
 
 if __name__ == "__main__":
-    logging.basicConfig(level=logging.INFO)
+    numeric_level = getattr(logging, os.getenv('LOG_LEVEL').upper(), None)
+    if not isinstance(numeric_level, int):
+        raise ValueError(f'Invalid log level: {os.getenv("LOG_LEVEL")}')
+    logging.basicConfig(level=numeric_level)
     logging.info("Starting up")
     runner = MediametadataToDB()
     runner.run()

mediametadatatodb_app/resources/MediametadataToDB.py

@@ -52,6 +52,19 @@
 #
 #  You should have received a copy of the GNU Affero General Public License
 #  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#  This program is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU Affero General Public License as published
+#  by the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  This program is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU Affero General Public License for more details.
+#
+#  You should have received a copy of the GNU Affero General Public License
+#  along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 import json
 import logging
@@ -209,6 +222,12 @@ def _get_access_status(graph):
     return 'unavailable'
 
 
+def _get_record_id(graph):
+    for resource in graph:
+        if '@type' in resource and resource['@type'] == 'https://www.ica.org/standards/RiC/ontology#Record':
+            return resource['@id'] if '@id' in resource else None
+
+
 def _create_sql_stmt(table_name, record, fields):
     db_fields = [dbField for dbField in fields
                  if dbField in record and record[dbField] is not None]
@@ -296,6 +315,8 @@ class MediametadataToDB:
                 consumer.poll(max_records=100)
                 for recordsJson in consumer:
                     records_json_data = recordsJson.value['@graph']
+                    record_id = _get_record_id(records_json_data)
+                    logging.debug(f'Processing record {record_id}')
                     access_status = _get_access_status(records_json_data)
                     if access_status == 'public' or access_status == 'closed':
                         for recordJsonData in records_json_data:
@@ -310,22 +331,24 @@ class MediametadataToDB:
                                                             _get_values_from_thumbnail_object,
                                                             access_status)
                     else:
-                        logging.info('Ignoring record since access of digitalObject is {}'
-                                     .format(access_status))
+                        logging.info(f'Ignoring record {record_id} since access of digitalObject is {access_status}')
+                        reporter.send_message(record_id,
+                                              "SUCCESS",
+                                              "Ignoring record since access of " +
+                                              "digitalObject is unavailable")
                 # arriving here means there are no new messages to poll from
                 _write_values_in_db(mariadb_cursor, record_values_for_db)
                 mariadb_connection.commit()
                 for record_value in record_values_for_db:
-                    # TODO: Create more meaningful reports!
+                    logging.info(f'Record {record_value["@id"]} successfully indexed')
                     reporter.send_message(record_value['@id'], "SUCCESS", "Indexing successful")
                 record_values_for_db = []
                 consumer.commit()
-        except KafkaError as ex:
-            status = 'It was not possible to consume the Kafka messages.' + '\n' + str(ex)
-            logging.error(status)
         except Exception as ex:
             status = 'It was not possible to consume the Kafka messages.' + '\n' + str(ex)
             logging.error(status)
+            for record_value in record_values_for_db:
+                reporter.send_message(record_value['@id'], "FAILURE", f"Indexing failed: {ex}")
 
     def __init__(self):
         # TODO : maybe take that to a configuration (development vs pod running in