Unverified Commit 5032939a authored by Sebastian Schüpbach's avatar Sebastian Schüpbach
Browse files
parent 5dd94ddb
Pipeline #22317 passed with stages
in 5 minutes and 21 seconds
...@@ -19,8 +19,11 @@ ...@@ -19,8 +19,11 @@
package ch.memobase package ch.memobase
import ch.memobase.models.{DeleteMessage, ParserIgnore, Report} import ch.memobase.models.{DeleteMessage, ParserIgnore, Report}
import org.apache.kafka.clients.consumer.KafkaConsumer
import org.apache.logging.log4j.scala.Logging import org.apache.logging.log4j.scala.Logging
import java.time.Duration
import scala.collection.JavaConverters._
import scala.util.{Failure, Success} import scala.util.{Failure, Success}
object App object App
...@@ -32,20 +35,30 @@ object App ...@@ -32,20 +35,30 @@ object App
with MsgFilter with MsgFilter
with Logging { with Logging {
lazy private val consumer = {
val consumer = new KafkaConsumer[String, String](consumerProps)
logger.debug(s"Subscribing to topic $inputTopic")
consumer.subscribe(List(inputTopic).asJava)
consumer
}
parse(args) match { parse(args) match {
case Some((sessionId, filters, dryRun)) => case Some((sessionId, filters, dryRun)) =>
try { try {
for (record <- poll) { while (true) {
Report(record) match { val records = consumer.poll(Duration.ofMillis(pollTimeout)).asScala
case Success(rep) => for (record <- records) {
if (filters.forall(f => f(rep))) { Report(record) match {
sendDelete(DeleteMessage(rep, sessionId), dryRun) case Success(rep) =>
} if (filters.forall(f => f(rep))) {
case Failure(_: ParserIgnore) => sendDelete(DeleteMessage(rep, sessionId), dryRun)
logger.debug("Ignoring irrelevant message") }
case Failure(ex) => case Failure(_: ParserIgnore) =>
logger.warn(s"Ignoring message because parsing failed: ${ex.getMessage}") logger.debug("Ignoring irrelevant message")
logger.info(s"${record.value()}") case Failure(ex) =>
logger.warn(s"Ignoring message because parsing failed: ${ex.getMessage}")
logger.info(s"${record.value()}")
}
} }
} }
} catch { } catch {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment