Unverified Commit 5dd94ddb authored by Sebastian Schüpbach's avatar Sebastian Schüpbach
Browse files

refactor as for-loop


Signed-off-by: Sebastian Schüpbach's avatarSebastian Schüpbach <sebastian.schuepbach@unibas.ch>
parent 4691604f
Pipeline #22313 passed with stages
in 4 minutes and 53 seconds
...@@ -21,7 +21,6 @@ package ch.memobase ...@@ -21,7 +21,6 @@ package ch.memobase
import ch.memobase.models.{DeleteMessage, ParserIgnore, Report} import ch.memobase.models.{DeleteMessage, ParserIgnore, Report}
import org.apache.logging.log4j.scala.Logging import org.apache.logging.log4j.scala.Logging
import scala.collection.immutable.HashSet
import scala.util.{Failure, Success} import scala.util.{Failure, Success}
object App object App
...@@ -35,41 +34,20 @@ object App ...@@ -35,41 +34,20 @@ object App
parse(args) match { parse(args) match {
case Some((sessionId, filters, dryRun)) => case Some((sessionId, filters, dryRun)) =>
var counter = 0
var matchCounter = 0
try { try {
Stream for (record <- poll) {
.continually(poll) Report(record) match {
.flatMap(records => { case Success(rep) =>
logger.info(s"Processing records batch with size ${records.size}; ${counter} processed so far") if (filters.forall(f => f(rep))) {
counter += records.size sendDelete(DeleteMessage(rep, sessionId), dryRun)
records
.flatMap { record =>
Report(record) match {
case Success(rep) => Some(rep)
case Failure(_: ParserIgnore) =>
logger.debug("Ignoring irrelevant message")
None
case Failure(ex) =>
logger.warn(s"Ignoring message because parsing failed: ${ex.getMessage}")
logger.info(s"${record.value()}")
None
}
} }
.filter(record => filters.forall(f => f(record))) case Failure(_: ParserIgnore) =>
.map(record => { logger.debug("Ignoring irrelevant message")
matchCounter += 1 case Failure(ex) =>
logger.info(s"${matchCounter} matches so far") logger.warn(s"Ignoring message because parsing failed: ${ex.getMessage}")
record logger.info(s"${record.value()}")
})
.foldLeft(HashSet[DeleteMessage]())((agg, record) => {
logger.info(s"Size of delete messages: ${agg.size}")
agg + DeleteMessage(record, sessionId)
}
)
} }
) }
.foreach(id => sendDelete(id, dryRun))
} catch { } catch {
case e: Exception => case e: Exception =>
logger.error(e) logger.error(e)
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment