Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
memoriav
Memobase 2020
services
Deletion Components
Import Process Delete
Commits
5a32dbd0
Unverified
Commit
5a32dbd0
authored
Nov 26, 2020
by
Sebastian Schüpbach
Browse files
don't build report object of irrelevant messages
Signed-off-by:
Sebastian Schüpbach
<
sebastian.schuepbach@unibas.ch
>
parent
62957f51
Pipeline
#17880
passed with stages
in 5 minutes and 26 seconds
Changes
4
Pipelines
1
Show whitespace changes
Inline
Side-by-side
src/main/scala/ch/memobase/App.scala
View file @
5a32dbd0
...
...
@@ -18,7 +18,7 @@
package
ch.memobase
import
ch.memobase.models.
{
DeleteMessage
,
Report
}
import
ch.memobase.models.
{
DeleteMessage
,
ParserIgnore
,
Report
}
import
org.apache.logging.log4j.scala.Logging
import
scala.collection.immutable.HashSet
...
...
@@ -44,6 +44,9 @@ object App
.
flatMap
{
record
=>
Report
(
record
)
match
{
case
Success
(
rep
)
=>
Some
(
rep
)
case
Failure
(
_:
ParserIgnore
)
=>
logger
.
debug
(
"Ignoring irrelevant message"
)
None
case
Failure
(
ex
)
=>
logger
.
warn
(
s
"Ignoring message because parsing failed: ${ex.getMessage}"
)
logger
.
info
(
s
"${record.value()}"
)
...
...
src/main/scala/ch/memobase/MsgFilter.scala
View file @
5a32dbd0
...
...
@@ -37,8 +37,6 @@ trait MsgFilter {
recordSets
:
Seq
[
String
],
records
:
Seq
[
String
],
sessions
:
Seq
[
String
])
:
Seq
[
FilterFun
]
=
Seq
(
buildStepFilter
(
"fedora-ingest"
))
++
Seq
(
buildStatusFilter
(
"SUCCESS"
))
++
Seq
(
buildCreatedAfterFilter
(
standardiseTimestamp
(
createdAfter
)))
++
Seq
(
buildCreatedBeforeFilter
(
standardiseTimestamp
(
createdBefore
)))
++
institutions
.
map
(
v
=>
buildInstitutionIdFilter
(
v
))
++
...
...
@@ -73,11 +71,4 @@ trait MsgFilter {
private
val
buildCreatedBeforeFilter
:
String
=>
FilterFun
=
timestamp
=>
report
=>
dateFormatter
.
parse
(
timestamp
).
before
(
report
.
timestamp
)
private
val
buildStatusFilter
:
String
=>
FilterFun
=
status
=>
report
=>
report
.
status
==
status
private
val
buildStepFilter
:
String
=>
FilterFun
=
step
=>
report
=>
report
.
step
==
step
}
src/main/scala/ch/memobase/models/ParserException.scala
View file @
5a32dbd0
...
...
@@ -21,3 +21,5 @@ package ch.memobase.models
import
java.io.IOException
class
ParserException
(
message
:
String
)
extends
IOException
(
message
)
class
ParserIgnore
(
message
:
String
)
extends
Exception
(
message
)
\ No newline at end of file
src/main/scala/ch/memobase/models/Report.scala
View file @
5a32dbd0
...
...
@@ -41,13 +41,19 @@ object Report {
def
apply
(
consumerRecord
:
ConsumerRecord
[
String
,
String
])
:
Try
[
Report
]
=
Try
{
val
json
=
Try
(
ujson
.
read
(
consumerRecord
.
value
())).
getOrElse
(
throw
new
ParserException
(
"JSON is not valid!"
))
val
id
=
Try
(
json
.
obj
(
"id"
).
str
).
getOrElse
(
throw
new
ParserException
(
"No `id` field in JSON obj"
))
val
step
=
Try
(
json
.
obj
(
"step"
).
str
).
getOrElse
(
throw
new
ParserException
(
"No `step` field in JSON obj"
))
if
(
step
!=
"fedora-ingest"
)
{
throw
new
ParserIgnore
(
"No fedora-ingest message"
)
}
val
status
=
Try
(
json
.
obj
(
"status"
).
str
).
getOrElse
(
throw
new
ParserException
(
"No `status` field in JSON obj"
))
if
(
status
!=
"SUCCESS"
)
{
throw
new
ParserIgnore
(
"No SUCCESS message"
)
}
val
id
=
Try
(
json
.
obj
(
"id"
).
str
).
getOrElse
(
throw
new
ParserException
(
"No `id` field in JSON obj"
))
val
timestampString
=
Try
(
json
.
obj
(
"timestamp"
).
str
).
getOrElse
(
throw
new
ParserException
(
"No `timestamp` field in JSON obj"
))
val
timestamp
=
Try
(
dateFormatter
.
parse
(
timestampString
))
.
orElse
(
Try
(
shortDateFormatter
.
parse
(
timestampString
)))
.
getOrElse
(
throw
new
ParserException
(
"No valid timestamp"
))
val
status
=
Try
(
json
.
obj
(
"status"
).
str
).
getOrElse
(
throw
new
ParserException
(
"No `status` field in JSON obj"
))
val
message
=
Try
(
json
.
obj
(
"message"
).
str
).
getOrElse
(
throw
new
ParserException
(
"No `message` field in JSON obj"
))
val
headers
=
consumerRecord
.
headers
()
val
recordSet
=
Try
(
new
String
(
headers
.
lastHeader
(
"recordSetId"
).
value
()))
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment