Unverified Commit 8f0595dd authored by Sebastian Schüpbach's avatar Sebastian Schüpbach
Browse files

filter relevant reports before applying custom filters

parent c8271c14
Pipeline #17806 passed with stages
in 4 minutes and 52 seconds
...@@ -77,24 +77,13 @@ trait ArgParser { ...@@ -77,24 +77,13 @@ trait ArgParser {
OParser.parse(parser, args, Args()) match { OParser.parse(parser, args, Args()) match {
case Some(config) => Some( case Some(config) => Some(
config.sessionId, config.sessionId,
Seq(createdAfterFilter(standardiseTimestamp(config.createdAfterFilter))) ++ buildFilters(config.createdAfterFilter,
Seq(createdBeforeFilter(standardiseTimestamp(config.createdBeforeFilter))) ++ config.createdBeforeFilter,
config.institutionFilters.map(v => institutionIdFilter(v)) ++ config.institutionFilters,
config.recordFilters.map(v => createRecordIdFilter(v)) ++ config.recordSetFilters,
config.recordSetFilters.map(v => createRecordIdFilter(v)) ++ config.recordFilters,
config.sessionFilters.map(v => sessionIdFilter(v))) config.sessionFilters))
case None => None case None => None
} }
} }
private def standardiseTimestamp(calendar: Calendar): String =
f"${calendar.get(Calendar.YEAR)}%04d-" +
f"${calendar.get(Calendar.MONTH)}%02d-" +
f"${calendar.get(Calendar.DAY_OF_MONTH)}%02dT" +
f"${calendar.get(Calendar.HOUR_OF_DAY)}%02d:" +
f"${calendar.get(Calendar.MINUTE)}%02d:" +
f"${calendar.get(Calendar.SECOND)}%02d." +
f"${calendar.get(Calendar.MILLISECOND)}%03d"
} }
...@@ -19,6 +19,7 @@ ...@@ -19,6 +19,7 @@
package ch.memobase package ch.memobase
import java.text.SimpleDateFormat import java.text.SimpleDateFormat
import java.util.Calendar
import ch.memobase.models.Report import ch.memobase.models.Report
import org.apache.kafka.clients.consumer.ConsumerRecord import org.apache.kafka.clients.consumer.ConsumerRecord
...@@ -31,7 +32,32 @@ trait MsgFilter { ...@@ -31,7 +32,32 @@ trait MsgFilter {
type FilterFun = ConsumerRecord[String, String] => Boolean type FilterFun = ConsumerRecord[String, String] => Boolean
val sessionIdFilter: String => FilterFun = def buildFilters(createdAfter: Calendar,
createdBefore: Calendar,
institutions: Seq[String],
recordSets: Seq[String],
records: Seq[String],
sessions: Seq[String]): Seq[FilterFun] =
Seq(buildStepFilter("fedora-ingest")) ++
Seq(buildStatusFilter("SUCCESS")) ++
Seq(buildCreatedAfterFilter(standardiseTimestamp(createdAfter))) ++
Seq(buildCreatedBeforeFilter(standardiseTimestamp(createdBefore))) ++
institutions.map(v => buildInstitutionIdFilter(v)) ++
records.map(v => buildRecordIdFilter(v)) ++
recordSets.map(v => buildRecordSetIdFilter(v)) ++
sessions.map(v => buildSessionIdFilter(v))
private def standardiseTimestamp(calendar: Calendar): String =
f"${calendar.get(Calendar.YEAR)}%04d-" +
f"${calendar.get(Calendar.MONTH)}%02d-" +
f"${calendar.get(Calendar.DAY_OF_MONTH)}%02dT" +
f"${calendar.get(Calendar.HOUR_OF_DAY)}%02d:" +
f"${calendar.get(Calendar.MINUTE)}%02d:" +
f"${calendar.get(Calendar.SECOND)}%02d." +
f"${calendar.get(Calendar.MILLISECOND)}%03d"
private val buildSessionIdFilter: String => FilterFun =
sessionId => rec => sessionId => rec =>
rec.headers() rec.headers()
.headers("sessionId") .headers("sessionId")
...@@ -39,7 +65,7 @@ trait MsgFilter { ...@@ -39,7 +65,7 @@ trait MsgFilter {
.map(header => new String(header.value())) .map(header => new String(header.value()))
.exists(v => v == sessionId) .exists(v => v == sessionId)
val recordSetIdFilter: String => FilterFun = private val buildRecordSetIdFilter: String => FilterFun =
recordSetId => rec => recordSetId => rec =>
rec.headers() rec.headers()
.headers("recordSetId") .headers("recordSetId")
...@@ -47,7 +73,7 @@ trait MsgFilter { ...@@ -47,7 +73,7 @@ trait MsgFilter {
.map(header => new String(header.value())) .map(header => new String(header.value()))
.exists(v => v == recordSetId) .exists(v => v == recordSetId)
val institutionIdFilter: String => FilterFun = private val buildInstitutionIdFilter: String => FilterFun =
institutionId => rec => institutionId => rec =>
rec.headers() rec.headers()
.headers("institutionId") .headers("institutionId")
...@@ -55,22 +81,28 @@ trait MsgFilter { ...@@ -55,22 +81,28 @@ trait MsgFilter {
.map(header => new String(header.value())) .map(header => new String(header.value()))
.exists(v => v == institutionId) .exists(v => v == institutionId)
val createRecordIdFilter: String => FilterFun = private val buildRecordIdFilter: String => FilterFun =
recordId => rec => recordId => rec =>
Report(rec.value()).id == recordId Report(rec.value()).id == recordId
val createdAfterFilter: String => FilterFun = private val buildCreatedAfterFilter: String => FilterFun =
timestamp => rec => { timestamp => rec => {
val recordTimestamp = Report(rec.value()).timestamp val recordTimestamp = Report(rec.value()).timestamp
recordTimestamp == timestamp || recordTimestamp == timestamp ||
dateFormatter.parse(timestamp).after(dateFormatter.parse(recordTimestamp)) dateFormatter.parse(timestamp).after(dateFormatter.parse(recordTimestamp))
} }
val createdBeforeFilter: String => FilterFun = private val buildCreatedBeforeFilter: String => FilterFun =
timestamp => rec => { timestamp => rec => {
val recordTimestamp = Report(rec.value()).timestamp val recordTimestamp = Report(rec.value()).timestamp
recordTimestamp == timestamp || recordTimestamp == timestamp ||
dateFormatter.parse(timestamp).before(dateFormatter.parse(recordTimestamp)) dateFormatter.parse(timestamp).before(dateFormatter.parse(recordTimestamp))
} }
private val buildStatusFilter: String => FilterFun =
status => rec => Report(rec.value()).status == status
private val buildStepFilter: String => FilterFun =
step => rec => Report(rec.value()).step == step
} }
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment