Due to a scheduled upgrade to version 14.10, GitLab will be unavailabe on Monday 30.05., from 19:00 until 20:00.

Unverified Commit 5dd94ddb authored by Sebastian Schüpbach's avatar Sebastian Schüpbach
Browse files

refactor as for-loop


Signed-off-by: Sebastian Schüpbach's avatarSebastian Schüpbach <sebastian.schuepbach@unibas.ch>
parent 4691604f
Pipeline #22313 passed with stages
in 4 minutes and 53 seconds
...@@ -21,7 +21,6 @@ package ch.memobase ...@@ -21,7 +21,6 @@ package ch.memobase
import ch.memobase.models.{DeleteMessage, ParserIgnore, Report} import ch.memobase.models.{DeleteMessage, ParserIgnore, Report}
import org.apache.logging.log4j.scala.Logging import org.apache.logging.log4j.scala.Logging
import scala.collection.immutable.HashSet
import scala.util.{Failure, Success} import scala.util.{Failure, Success}
object App object App
...@@ -35,41 +34,20 @@ object App ...@@ -35,41 +34,20 @@ object App
parse(args) match { parse(args) match {
case Some((sessionId, filters, dryRun)) => case Some((sessionId, filters, dryRun)) =>
var counter = 0
var matchCounter = 0
try { try {
Stream for (record <- poll) {
.continually(poll)
.flatMap(records => {
logger.info(s"Processing records batch with size ${records.size}; ${counter} processed so far")
counter += records.size
records
.flatMap { record =>
Report(record) match { Report(record) match {
case Success(rep) => Some(rep) case Success(rep) =>
if (filters.forall(f => f(rep))) {
sendDelete(DeleteMessage(rep, sessionId), dryRun)
}
case Failure(_: ParserIgnore) => case Failure(_: ParserIgnore) =>
logger.debug("Ignoring irrelevant message") logger.debug("Ignoring irrelevant message")
None
case Failure(ex) => case Failure(ex) =>
logger.warn(s"Ignoring message because parsing failed: ${ex.getMessage}") logger.warn(s"Ignoring message because parsing failed: ${ex.getMessage}")
logger.info(s"${record.value()}") logger.info(s"${record.value()}")
None
}
}
.filter(record => filters.forall(f => f(record)))
.map(record => {
matchCounter += 1
logger.info(s"${matchCounter} matches so far")
record
})
.foldLeft(HashSet[DeleteMessage]())((agg, record) => {
logger.info(s"Size of delete messages: ${agg.size}")
agg + DeleteMessage(record, sessionId)
} }
)
} }
)
.foreach(id => sendDelete(id, dryRun))
} catch { } catch {
case e: Exception => case e: Exception =>
logger.error(e) logger.error(e)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment