Unverified Commit 98e136b9 authored by Sebastian Schüpbach's avatar Sebastian Schüpbach
Browse files

check report validity beforehand

parent 8f98a84e
Pipeline #17860 passed with stages
in 4 minutes and 47 seconds
...@@ -18,10 +18,11 @@ ...@@ -18,10 +18,11 @@
package ch.memobase package ch.memobase
import ch.memobase.models.DeleteMessage import ch.memobase.models.{DeleteMessage, Report}
import org.apache.logging.log4j.scala.Logging import org.apache.logging.log4j.scala.Logging
import scala.collection.immutable.HashSet import scala.collection.immutable.HashSet
import scala.util.{Failure, Success}
object App object App
extends KProducer extends KProducer
...@@ -40,6 +41,15 @@ object App ...@@ -40,6 +41,15 @@ object App
.takeWhile(records => records.nonEmpty) .takeWhile(records => records.nonEmpty)
.flatMap(records => .flatMap(records =>
records records
.flatMap { record =>
Report(record) match {
case Success(rep) => Some(rep)
case Failure(ex) =>
logger.warn(s"Ignoring message because parsing failed: ${ex.getMessage}")
logger.info(s"$record")
None
}
}
.filter(record => filters.forall(f => f(record))) .filter(record => filters.forall(f => f(record)))
.foldLeft(HashSet[DeleteMessage]())((agg, record) => agg + DeleteMessage(record, sessionId)) .foldLeft(HashSet[DeleteMessage]())((agg, record) => agg + DeleteMessage(record, sessionId))
) )
......
...@@ -22,18 +22,14 @@ import java.text.SimpleDateFormat ...@@ -22,18 +22,14 @@ import java.text.SimpleDateFormat
import java.util.Calendar import java.util.Calendar
import ch.memobase.models.Report import ch.memobase.models.Report
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.apache.logging.log4j.scala.Logging import org.apache.logging.log4j.scala.Logging
import scala.collection.JavaConverters._
import scala.util.{Failure, Success, Try}
trait MsgFilter { trait MsgFilter {
self: Logging => self: Logging =>
private val dateFormatter = new SimpleDateFormat("YYYY-MM-dd'T'HH:mm:ss.SSS") private val dateFormatter = new SimpleDateFormat("YYYY-MM-dd'T'HH:mm:ss.SSS")
type FilterFun = ConsumerRecord[String, String] => Boolean type FilterFun = Report => Boolean
def buildFilters(createdAfter: Calendar, def buildFilters(createdAfter: Calendar,
createdBefore: Calendar, createdBefore: Calendar,
...@@ -61,60 +57,27 @@ trait MsgFilter { ...@@ -61,60 +57,27 @@ trait MsgFilter {
f"${calendar.get(Calendar.MILLISECOND)}%03d" f"${calendar.get(Calendar.MILLISECOND)}%03d"
private val buildSessionIdFilter: String => FilterFun = private val buildSessionIdFilter: String => FilterFun =
sessionId => rec => sessionId => report => report.sessionId == sessionId
rec.headers()
.headers("sessionId")
.asScala
.map(header => new String(header.value()))
.exists(v => v == sessionId)
private val buildRecordSetIdFilter: String => FilterFun = private val buildRecordSetIdFilter: String => FilterFun =
recordSetId => rec => recordSetId => report => report.recordSetId == recordSetId
rec.headers()
.headers("recordSetId")
.asScala
.map(header => new String(header.value()))
.exists(v => v == recordSetId)
private val buildInstitutionIdFilter: String => FilterFun = private val buildInstitutionIdFilter: String => FilterFun =
institutionId => rec => institutionId => report => report.institutionId == institutionId
rec.headers()
.headers("institutionId")
.asScala
.map(header => new String(header.value()))
.exists(v => v == institutionId)
private val buildRecordIdFilter: String => FilterFun = private val buildRecordIdFilter: String => FilterFun =
recordId => rec => recordId => report => report.recordId == recordId
Report(rec.value()).id == recordId
private val buildCreatedAfterFilter: String => FilterFun = private val buildCreatedAfterFilter: String => FilterFun =
timestamp => rec => timestamp => report => dateFormatter.parse(timestamp).after(report.timestamp)
Try {
dateFormatter.parse(timestamp).after(dateFormatter.parse(Report(rec.value()).timestamp))
} match {
case Success(res) => res
case Failure(_) =>
logger.warn(s"Parsing of timestamp ${Report(rec.value()).timestamp} failed! Ignoring record")
false
}
private val buildCreatedBeforeFilter: String => FilterFun = private val buildCreatedBeforeFilter: String => FilterFun =
timestamp => rec => timestamp => report => dateFormatter.parse(timestamp).before(report.timestamp)
Try {
dateFormatter.parse(timestamp).before(dateFormatter.parse(Report(rec.value()).timestamp))
} match {
case Success(res) => res
case Failure(_) =>
logger.warn(s"Parsing of timestamp ${Report(rec.value()).timestamp} failed! Ignoring record")
false
}
private val buildStatusFilter: String => FilterFun = private val buildStatusFilter: String => FilterFun =
status => rec => Report(rec.value()).status == status status => report => report.status == status
private val buildStepFilter: String => FilterFun = private val buildStepFilter: String => FilterFun =
step => rec => Report(rec.value()).step == step step => report => report.step == step
} }
...@@ -18,16 +18,9 @@ ...@@ -18,16 +18,9 @@
package ch.memobase.models package ch.memobase.models
import org.apache.kafka.clients.consumer.ConsumerRecord
import scala.collection.JavaConverters._
case class DeleteMessage(recordId: String, recordSetId: String, institutionId: String, sessionId: String) case class DeleteMessage(recordId: String, recordSetId: String, institutionId: String, sessionId: String)
object DeleteMessage { object DeleteMessage {
def apply(consumerRecord: ConsumerRecord[String, String], sessionId: String): DeleteMessage = { def apply(report: Report, sessionId: String): DeleteMessage =
val recordSetId = new String(consumerRecord.headers().headers("recordSetId").asScala.head.value()) DeleteMessage(report.msgKey, report.recordSetId, report.institutionId, report.sessionId)
val institutionId = new String(consumerRecord.headers().headers("institutionId").asScala.head.value())
DeleteMessage(consumerRecord.key(), recordSetId, institutionId, sessionId)
}
} }
...@@ -18,18 +18,38 @@ ...@@ -18,18 +18,38 @@
package ch.memobase.models package ch.memobase.models
import upickle.default.{Reader, _} import java.text.SimpleDateFormat
import java.util.Date
case class Report(id: String, import org.apache.kafka.clients.consumer.ConsumerRecord
import scala.util.Try
case class Report(msgKey: String,
recordId: String,
step: String, step: String,
timestamp: String, timestamp: Date,
status: String, status: String,
message: String) message: String,
recordSetId: String,
institutionId: String,
sessionId: String)
object Report { object Report {
def apply(json: String): Report = { private val dateFormatter = new SimpleDateFormat("YYYY-MM-dd'T'HH:mm:ss.SSS")
read[Report](json)
def apply(consumerRecord: ConsumerRecord[String, String]): Try[Report] = Try {
val json = ujson.read(consumerRecord.value())
val id = json.obj("id").str
val step = json.obj("step").str
val timestamp = dateFormatter.parse(json.obj("timestamp").str)
val status = json.obj("status").str
val message = json.obj("message").str
val headers = consumerRecord.headers()
val recordSet = new String(headers.lastHeader("recordSetId").value())
val institution = new String(headers.lastHeader("institutionId").value())
val session = new String(headers.lastHeader("sessionId").value())
Report(consumerRecord.key, id, step, timestamp, status, message, recordSet, institution, session)
} }
implicit val reader: Reader[Report] = macroR
} }
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment