more logging; get reports from normalization-service

Signed-off-by: Sebastian Schüpbach <sebastian.schuepbach@unibas.ch>

src/main/scala/ch/memobase/App.scala

@@ -35,12 +35,15 @@ object App
 
   parse(args) match {
     case Some((sessionId, filters, dryRun)) =>
+      var counter = 0
+      var matchCounter = 0
       try {
         Stream
           .continually(poll)
           .takeWhile(records => records.nonEmpty)
           .flatMap(records => {
-            logger.info("Processing records batch with size ${records.size}")
+            logger.info(s"Processing records batch with size ${records.size}; ${counter} processed so far")
+            counter += records.size
             records
               .flatMap { record =>
                 Report(record) match {
@@ -55,6 +58,11 @@ object App
                 }
               }
               .filter(record => filters.forall(f => f(record)))
+              .map(record => {
+                matchCounter += 1
+                logger.info(s"${matchCounter} matches so far")
+                record
+              })
               .foldLeft(HashSet[DeleteMessage]())((agg, record) => {
                 logger.info(s"Size of delete messages: ${agg.size}")
                 agg + DeleteMessage(record, sessionId)

src/main/scala/ch/memobase/models/Report.scala

 /*
  * Import Process Delete
- * Copyright (C) 2020 Memoriav
+ * Copyright (C) 2021 Memoriav
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as published by
@@ -18,11 +18,10 @@
 
 package ch.memobase.models
 
-import java.text.SimpleDateFormat
-import java.util.Date
-
 import org.apache.kafka.clients.consumer.ConsumerRecord
 
+import java.text.SimpleDateFormat
+import java.util.Date
 import scala.util.Try
 
 case class Report(msgKey: String,
@@ -42,8 +41,8 @@ object Report {
   def apply(consumerRecord: ConsumerRecord[String, String]): Try[Report] = Try {
     val json = Try(ujson.read(consumerRecord.value())).getOrElse(throw new ParserException("JSON is not valid!"))
     val step = Try(json.obj("step").str).getOrElse(throw new ParserException("No `step` field in JSON obj"))
-    if (step != "fedora-ingest") {
-      throw new ParserIgnore("No fedora-ingest message")
+    if (step != "normalization-service") {
+      throw new ParserIgnore("No normalization-service message")
     }
     val status = Try(json.obj("status").str).getOrElse(throw new ParserException("No `status` field in JSON obj"))
     if (status != "SUCCESS") {