Unverified Commit 5dd94ddb authored by Sebastian Schüpbach's avatar Sebastian Schüpbach
Browse files

refactor as for-loop


Signed-off-by: Sebastian Schüpbach's avatarSebastian Schüpbach <sebastian.schuepbach@unibas.ch>
parent 4691604f
Pipeline #22313 passed with stages
in 4 minutes and 53 seconds
......@@ -21,7 +21,6 @@ package ch.memobase
import ch.memobase.models.{DeleteMessage, ParserIgnore, Report}
import org.apache.logging.log4j.scala.Logging
import scala.collection.immutable.HashSet
import scala.util.{Failure, Success}
object App
......@@ -35,41 +34,20 @@ object App
parse(args) match {
case Some((sessionId, filters, dryRun)) =>
var counter = 0
var matchCounter = 0
try {
Stream
.continually(poll)
.flatMap(records => {
logger.info(s"Processing records batch with size ${records.size}; ${counter} processed so far")
counter += records.size
records
.flatMap { record =>
Report(record) match {
case Success(rep) => Some(rep)
case Failure(_: ParserIgnore) =>
logger.debug("Ignoring irrelevant message")
None
case Failure(ex) =>
logger.warn(s"Ignoring message because parsing failed: ${ex.getMessage}")
logger.info(s"${record.value()}")
None
}
for (record <- poll) {
Report(record) match {
case Success(rep) =>
if (filters.forall(f => f(rep))) {
sendDelete(DeleteMessage(rep, sessionId), dryRun)
}
.filter(record => filters.forall(f => f(record)))
.map(record => {
matchCounter += 1
logger.info(s"${matchCounter} matches so far")
record
})
.foldLeft(HashSet[DeleteMessage]())((agg, record) => {
logger.info(s"Size of delete messages: ${agg.size}")
agg + DeleteMessage(record, sessionId)
}
)
case Failure(_: ParserIgnore) =>
logger.debug("Ignoring irrelevant message")
case Failure(ex) =>
logger.warn(s"Ignoring message because parsing failed: ${ex.getMessage}")
logger.info(s"${record.value()}")
}
)
.foreach(id => sendDelete(id, dryRun))
}
} catch {
case e: Exception =>
logger.error(e)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment