Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
memoriav
Memobase 2020
services
Deletion Components
Import Process Delete
Commits
8f0595dd
Unverified
Commit
8f0595dd
authored
Nov 25, 2020
by
Sebastian Schüpbach
Browse files
filter relevant reports before applying custom filters
parent
c8271c14
Pipeline
#17806
passed with stages
in 4 minutes and 52 seconds
Changes
2
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
src/main/scala/ch/memobase/ArgParser.scala
View file @
8f0595dd
...
...
@@ -77,24 +77,13 @@ trait ArgParser {
OParser
.
parse
(
parser
,
args
,
Args
())
match
{
case
Some
(
config
)
=>
Some
(
config
.
sessionId
,
Seq
(
createdAfterFilter
(
standardiseTimestamp
(
config
.
createdAfterFilter
)))
++
Seq
(
createdBeforeFilter
(
standardiseTimestamp
(
config
.
createdBeforeFilter
)))
++
config
.
institutionFilters
.
map
(
v
=>
institutionIdFilter
(
v
))
++
config
.
record
Filters
.
map
(
v
=>
createRecordIdFilter
(
v
))
++
config
.
record
Set
Filters
.
map
(
v
=>
createRecordIdFilter
(
v
))
++
config
.
sessionFilters
.
map
(
v
=>
sessionIdFilter
(
v
)
))
buildFilters
(
config
.
createdAfterFilter
,
config
.
createdBeforeFilter
,
config
.
institutionFilters
,
config
.
record
SetFilters
,
config
.
recordFilters
,
config
.
sessionFilters
))
case
None
=>
None
}
}
private
def
standardiseTimestamp
(
calendar
:
Calendar
)
:
String
=
f
"${calendar.get(Calendar.YEAR)}%04d-"
+
f
"${calendar.get(Calendar.MONTH)}%02d-"
+
f
"${calendar.get(Calendar.DAY_OF_MONTH)}%02dT"
+
f
"${calendar.get(Calendar.HOUR_OF_DAY)}%02d:"
+
f
"${calendar.get(Calendar.MINUTE)}%02d:"
+
f
"${calendar.get(Calendar.SECOND)}%02d."
+
f
"${calendar.get(Calendar.MILLISECOND)}%03d"
}
src/main/scala/ch/memobase/MsgFilter.scala
View file @
8f0595dd
...
...
@@ -19,6 +19,7 @@
package
ch.memobase
import
java.text.SimpleDateFormat
import
java.util.Calendar
import
ch.memobase.models.Report
import
org.apache.kafka.clients.consumer.ConsumerRecord
...
...
@@ -31,7 +32,32 @@ trait MsgFilter {
type
FilterFun
=
ConsumerRecord
[
String
,
String
]
=>
Boolean
val
sessionIdFilter
:
String
=>
FilterFun
=
def
buildFilters
(
createdAfter
:
Calendar
,
createdBefore
:
Calendar
,
institutions
:
Seq
[
String
],
recordSets
:
Seq
[
String
],
records
:
Seq
[
String
],
sessions
:
Seq
[
String
])
:
Seq
[
FilterFun
]
=
Seq
(
buildStepFilter
(
"fedora-ingest"
))
++
Seq
(
buildStatusFilter
(
"SUCCESS"
))
++
Seq
(
buildCreatedAfterFilter
(
standardiseTimestamp
(
createdAfter
)))
++
Seq
(
buildCreatedBeforeFilter
(
standardiseTimestamp
(
createdBefore
)))
++
institutions
.
map
(
v
=>
buildInstitutionIdFilter
(
v
))
++
records
.
map
(
v
=>
buildRecordIdFilter
(
v
))
++
recordSets
.
map
(
v
=>
buildRecordSetIdFilter
(
v
))
++
sessions
.
map
(
v
=>
buildSessionIdFilter
(
v
))
private
def
standardiseTimestamp
(
calendar
:
Calendar
)
:
String
=
f
"${calendar.get(Calendar.YEAR)}%04d-"
+
f
"${calendar.get(Calendar.MONTH)}%02d-"
+
f
"${calendar.get(Calendar.DAY_OF_MONTH)}%02dT"
+
f
"${calendar.get(Calendar.HOUR_OF_DAY)}%02d:"
+
f
"${calendar.get(Calendar.MINUTE)}%02d:"
+
f
"${calendar.get(Calendar.SECOND)}%02d."
+
f
"${calendar.get(Calendar.MILLISECOND)}%03d"
private
val
buildSessionIdFilter
:
String
=>
FilterFun
=
sessionId
=>
rec
=>
rec
.
headers
()
.
headers
(
"sessionId"
)
...
...
@@ -39,7 +65,7 @@ trait MsgFilter {
.
map
(
header
=>
new
String
(
header
.
value
()))
.
exists
(
v
=>
v
==
sessionId
)
val
r
ecordSetIdFilter
:
String
=>
FilterFun
=
private
val
buildR
ecordSetIdFilter
:
String
=>
FilterFun
=
recordSetId
=>
rec
=>
rec
.
headers
()
.
headers
(
"recordSetId"
)
...
...
@@ -47,7 +73,7 @@ trait MsgFilter {
.
map
(
header
=>
new
String
(
header
.
value
()))
.
exists
(
v
=>
v
==
recordSetId
)
val
i
nstitutionIdFilter
:
String
=>
FilterFun
=
private
val
buildI
nstitutionIdFilter
:
String
=>
FilterFun
=
institutionId
=>
rec
=>
rec
.
headers
()
.
headers
(
"institutionId"
)
...
...
@@ -55,22 +81,28 @@ trait MsgFilter {
.
map
(
header
=>
new
String
(
header
.
value
()))
.
exists
(
v
=>
v
==
institutionId
)
val
create
RecordIdFilter
:
String
=>
FilterFun
=
private
val
build
RecordIdFilter
:
String
=>
FilterFun
=
recordId
=>
rec
=>
Report
(
rec
.
value
()).
id
==
recordId
val
c
reatedAfterFilter
:
String
=>
FilterFun
=
private
val
buildC
reatedAfterFilter
:
String
=>
FilterFun
=
timestamp
=>
rec
=>
{
val
recordTimestamp
=
Report
(
rec
.
value
()).
timestamp
recordTimestamp
==
timestamp
||
dateFormatter
.
parse
(
timestamp
).
after
(
dateFormatter
.
parse
(
recordTimestamp
))
}
val
c
reatedBeforeFilter
:
String
=>
FilterFun
=
private
val
buildC
reatedBeforeFilter
:
String
=>
FilterFun
=
timestamp
=>
rec
=>
{
val
recordTimestamp
=
Report
(
rec
.
value
()).
timestamp
recordTimestamp
==
timestamp
||
dateFormatter
.
parse
(
timestamp
).
before
(
dateFormatter
.
parse
(
recordTimestamp
))
}
private
val
buildStatusFilter
:
String
=>
FilterFun
=
status
=>
rec
=>
Report
(
rec
.
value
()).
status
==
status
private
val
buildStepFilter
:
String
=>
FilterFun
=
step
=>
rec
=>
Report
(
rec
.
value
()).
step
==
step
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment