Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
memoriav
Memobase 2020
services
Deletion Components
Import Process Delete
Commits
d53895cb
Unverified
Commit
d53895cb
authored
Feb 25, 2021
by
Sebastian Schüpbach
Browse files
use import-process-bridge reports
Signed-off-by:
Sebastian Schüpbach
<
sebastian.schuepbach@unibas.ch
>
parent
eae23e26
Pipeline
#22374
passed with stages
in 5 minutes and 15 seconds
Changes
3
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
k8s-manifests/job.yml
View file @
d53895cb
...
...
@@ -35,9 +35,9 @@ spec:
-
name
:
CLIENT_ID
value
:
import-process-delete
-
name
:
TOPIC_IN
value
:
import-
process-reporting
value
:
post
process
ing
-reporting
-
name
:
TOPIC_OUT
value
:
import-process-test
value
:
import-process-test
-delete
-
name
:
POLL_TIMEOUT
value
:
"
60000"
restartPolicy
:
Never
...
...
src/main/scala/ch/memobase/App.scala
View file @
d53895cb
...
...
@@ -21,7 +21,6 @@ package ch.memobase
import
ch.memobase.models.
{
DeleteMessage
,
ParserIgnore
,
Report
}
import
org.apache.logging.log4j.scala.Logging
import
scala.collection.mutable
import
scala.util.
{
Failure
,
Success
}
object
App
...
...
@@ -36,7 +35,6 @@ object App
parse
(
args
)
match
{
case
Some
((
sessionId
,
filters
,
dryRun
))
=>
try
{
val
recordSetIds
:
mutable.HashSet
[
String
]
=
mutable
.
HashSet
()
val
matches
=
Iterator
.
continually
(
poll
)
.
takeWhile
(
_
.
nonEmpty
)
.
flatten
...
...
@@ -50,10 +48,6 @@ object App
logger
.
info
(
"Match found"
)
List
(
DeleteMessage
(
rep
,
sessionId
))
case
Success
(
r
)
=>
if
(!
recordSetIds
.
contains
(
r
.
recordSetId
))
{
recordSetIds
+=
r
.
recordSetId
logger
.
info
(
r
.
recordSetId
)
}
List
()
case
Failure
(
_:
ParserIgnore
)
=>
logger
.
debug
(
"Ignoring irrelevant message"
)
...
...
src/main/scala/ch/memobase/models/Report.scala
View file @
d53895cb
...
...
@@ -41,12 +41,12 @@ object Report {
def
apply
(
consumerRecord
:
ConsumerRecord
[
String
,
String
])
:
Try
[
Report
]
=
Try
{
val
json
=
Try
(
ujson
.
read
(
consumerRecord
.
value
())).
getOrElse
(
throw
new
ParserException
(
"JSON is not valid!"
))
val
step
=
Try
(
json
.
obj
(
"step"
).
str
).
getOrElse
(
throw
new
ParserException
(
"No `step` field in JSON obj"
))
if
(
step
!=
"
normalization-servic
e"
)
{
if
(
step
!=
"
import-process-bridg
e"
)
{
throw
new
ParserIgnore
(
"No normalization-service message"
)
}
val
status
=
Try
(
json
.
obj
(
"status"
).
str
).
getOrElse
(
throw
new
ParserException
(
"No `status` field in JSON obj"
))
if
(
status
!
=
"
SUCCESS
"
)
{
throw
new
ParserIgnore
(
"
No SUCCESS message
"
)
if
(
status
=
=
"
FATAL
"
)
{
throw
new
ParserIgnore
(
"
FATAL message: No record forwarded, therefore ignoring report
"
)
}
val
id
=
Try
(
json
.
obj
(
"id"
).
str
).
getOrElse
(
throw
new
ParserException
(
"No `id` field in JSON obj"
))
val
timestampString
=
Try
(
json
.
obj
(
"timestamp"
).
str
).
getOrElse
(
throw
new
ParserException
(
"No `timestamp` field in JSON obj"
))
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment