Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
memoriav
Memobase 2020
services
Import Process
Media-Linker
Commits
684c0a96
Unverified
Commit
684c0a96
authored
Oct 23, 2020
by
Sebastian Schüpbach
Browse files
catch model build exceptions
parent
a098d053
Pipeline
#16014
passed with stages
in 4 minutes and 58 seconds
Changes
1
Pipelines
2
Hide whitespace changes
Inline
Side-by-side
src/main/kotlin/KafkaTopology.kt
View file @
684c0a96
...
@@ -24,6 +24,7 @@ import org.apache.jena.rdf.model.Model
...
@@ -24,6 +24,7 @@ import org.apache.jena.rdf.model.Model
import
org.apache.jena.rdf.model.ModelFactory
import
org.apache.jena.rdf.model.ModelFactory
import
org.apache.jena.rdf.model.Resource
import
org.apache.jena.rdf.model.Resource
import
org.apache.jena.rdf.model.ResourceFactory
import
org.apache.jena.rdf.model.ResourceFactory
import
org.apache.jena.riot.RiotException
import
org.apache.kafka.streams.StreamsBuilder
import
org.apache.kafka.streams.StreamsBuilder
import
org.apache.kafka.streams.kstream.Predicate
import
org.apache.kafka.streams.kstream.Predicate
import
org.memobase.rdf.EBUCORE
import
org.memobase.rdf.EBUCORE
...
@@ -50,10 +51,26 @@ class KafkaTopology(private val settings: SettingsLoader) {
...
@@ -50,10 +51,26 @@ class KafkaTopology(private val settings: SettingsLoader) {
val
stream
=
builder
.
stream
<
String
,
String
>(
settings
.
inputTopic
)
val
stream
=
builder
.
stream
<
String
,
String
>(
settings
.
inputTopic
)
val
instantiationBranch
=
stream
val
model
=
stream
.
transformValues
(
HeaderExtractionTransformSupplier
<
String
>())
.
transformValues
(
HeaderExtractionTransformSupplier
<
String
>())
.
mapValues
{
value
->
createModel
(
value
)
}
.
mapValues
{
value
->
createModel
(
value
)
}
.
mapValues
{
value
->
extractSubjects
(
value
)
}
.
branch
(
Predicate
{
_
,
value
->
value
!=
null
},
Predicate
{
_
,
_
->
true
}
)
model
[
1
]
.
mapValues
{
key
,
_
->
Report
(
key
,
ReportStatus
.
failure
,
"Can't create model. Is there anything wrong with the data?"
)
}
.
to
(
reportingTopic
)
val
instantiationBranch
=
model
[
0
]
.
mapValues
{
value
->
extractSubjects
(
value
!!
)
}
.
mapValues
{
readOnlyKey
,
value
->
.
mapValues
{
readOnlyKey
,
value
->
enrichSftpLocator
(
enrichSftpLocator
(
readOnlyKey
,
readOnlyKey
,
...
@@ -170,9 +187,13 @@ class KafkaTopology(private val settings: SettingsLoader) {
...
@@ -170,9 +187,13 @@ class KafkaTopology(private val settings: SettingsLoader) {
}
}
}
}
private
fun
createModel
(
data
:
Pair
<
String
,
HeaderMetadata
>):
Pair
<
Model
,
HeaderMetadata
>
{
private
fun
createModel
(
data
:
Pair
<
String
,
HeaderMetadata
>):
Pair
<
Model
,
HeaderMetadata
>
?
{
val
model
=
ModelFactory
.
createDefaultModel
()
val
model
=
ModelFactory
.
createDefaultModel
()
model
.
read
(
StringReader
(
data
.
first
),
""
,
Constant
.
rdfParserLang
)
try
{
model
.
read
(
StringReader
(
data
.
first
),
""
,
Constant
.
rdfParserLang
)
}
catch
(
ex
:
RiotException
)
{
return
null
}
return
Pair
(
model
,
data
.
second
)
return
Pair
(
model
,
data
.
second
)
}
}
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment