Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
memoriav
Memobase 2020
services
Import Process
Media-Linker
Commits
f7836b3c
Commit
f7836b3c
authored
Jun 12, 2020
by
Jonas Waeber
Browse files
Fix tests
parent
8bb963ed
Changes
4
Hide whitespace changes
Inline
Side-by-side
src/main/kotlin/KafkaTopology.kt
View file @
f7836b3c
...
...
@@ -19,12 +19,13 @@
package
org.memobase
import
com.beust.klaxon.Klaxon
import
java.io.
ByteArrayInputStream
import
java.io.
StringReader
import
java.io.StringWriter
import
kotlin.system.exitProcess
import
org.apache.jena.rdf.model.Model
import
org.apache.jena.rdf.model.ModelFactory
import
org.apache.jena.rdf.model.Resource
import
org.apache.jena.rdf.model.Statement
import
org.apache.jena.rdf.model.impl.StatementImpl
import
org.apache.kafka.streams.StreamsBuilder
import
org.apache.kafka.streams.Topology
...
...
@@ -60,7 +61,7 @@ class KafkaTopology(private val settings: SettingsLoader) {
fun
build
():
Topology
{
val
builder
=
StreamsBuilder
()
val
stream
=
builder
.
stream
<
String
,
ByteArray
>(
settings
.
inputTopic
)
val
stream
=
builder
.
stream
<
String
,
String
>(
settings
.
inputTopic
)
val
instantiationBranch
=
stream
.
mapValues
{
value
->
createModel
(
value
)
}
...
...
@@ -74,7 +75,7 @@ class KafkaTopology(private val settings: SettingsLoader) {
.
mapValues
{
value
->
enrichSftpLocator
(
value
)
}
.
mapValues
{
value
->
val
out
=
StringWriter
()
value
.
first
.
write
(
out
)
value
.
first
.
write
(
out
,
"NTRIPLES"
)
out
.
toString
()
}
...
...
@@ -96,7 +97,7 @@ class KafkaTopology(private val settings: SettingsLoader) {
val
unchangedWrittenResources
=
instantiationBranch
[
1
]
.
mapValues
{
value
->
val
out
=
StringWriter
()
value
.
first
.
write
(
out
)
value
.
first
.
write
(
out
,
"NTRIPLES"
)
out
.
toString
()
}
...
...
@@ -118,15 +119,15 @@ class KafkaTopology(private val settings: SettingsLoader) {
return
res
.
hasProperty
(
RICO
.
type
,
"digitalObject"
)
&&
!
res
.
hasProperty
(
EBUCORE
.
locator
)
}
private
fun
createModel
(
data
:
ByteArray
):
Model
{
private
fun
createModel
(
data
:
String
):
Model
{
val
model
=
ModelFactory
.
createDefaultModel
()
model
.
read
(
ByteArrayInputStream
(
data
),
""
,
"NTRIPLES"
)
model
.
read
(
StringReader
(
data
),
""
,
"NTRIPLES"
)
return
model
}
private
fun
enrichSftpLocator
(
data
:
Pair
<
Model
,
Resource
>):
Pair
<
Model
,
Report
>
{
var
isEnriched
=
false
var
link
=
""
val
addedStatements
=
mutableListOf
<
Statement
>()
for
(
stmt
in
data
.
second
.
listProperties
(
RICO
.
identifiedBy
))
{
if
(
stmt
!=
null
)
{
if
(
stmt
.
`object`
.
isResource
)
{
...
...
@@ -135,23 +136,23 @@ class KafkaTopology(private val settings: SettingsLoader) {
val
value
=
resource
.
getProperty
(
RICO
.
identifier
).
literal
for
(
file
in
files
)
{
if
(
file
.
contains
(
value
.
string
))
{
link
=
"sftp://$file"
link
=
"sftp://$
{settings.sftpSettings.host}:${settings.sftpSettings.port}:$
file"
val
literal
=
data
.
first
.
createLiteral
(
link
)
data
.
first
.
add
(
StatementImpl
(
data
.
second
,
EBUCORE
.
locator
,
literal
))
isEnriched
=
true
addedStatements
.
add
(
StatementImpl
(
data
.
second
,
EBUCORE
.
locator
,
literal
))
}
}
}
}
}
}
if
(
isEnriched
)
{
if
(
addedStatements
.
size
>
0
)
{
data
.
first
.
add
(
addedStatements
)
return
Pair
(
data
.
first
,
Report
(
data
.
second
.
id
.
labelString
,
data
.
second
.
uri
,
ReportStatus
.
success
,
ReportMessages
.
reportSuccess
(
data
.
second
.
id
.
labelString
,
link
)
ReportMessages
.
reportSuccess
(
data
.
second
.
uri
,
link
)
)
)
}
else
{
...
...
src/test/kotlin/Tests.kt
View file @
f7836b3c
...
...
@@ -38,6 +38,8 @@ import org.memobase.testing.EmbeddedSftpServer
class
Tests
{
private
val
log
=
LogManager
.
getLogger
(
"TestLogger"
)
private
val
regex
=
Regex
(
"(_:B[A-Za-z0-9]+)"
)
private
val
resourcePath
=
"src/test/resources/data"
private
fun
readFile
(
fileName
:
String
):
String
{
return
File
(
"$resourcePath/$fileName"
).
readText
(
Charset
.
defaultCharset
())
...
...
@@ -81,10 +83,19 @@ class Tests {
var
count
=
0
while
(
record
!=
null
)
{
count
+=
1
val
sortedResult
=
record
.
value
().
lines
().
map
{
var
replacedString
=
it
for
(
matchResult
in
regex
.
findAll
(
it
))
{
replacedString
=
replacedString
.
replace
(
matchResult
.
groups
[
0
]
?.
value
.
orEmpty
(),
"_:B"
)
}
replacedString
}.
sorted
().
reduce
{
acc
,
s
->
acc
+
"\n"
+
s
}
assertThat
(
record
)
.
isNotNull
.
hasFieldOrPropertyWithValue
(
"key"
,
params
.
expectedOutputKey
)
.
hasFieldOrPropertyWithValue
(
"value"
,
readFile
(
params
.
expectedOutputFile
))
assertThat
(
sortedResult
).
isEqualTo
(
readFile
(
params
.
expectedOutputFile
))
val
reportedRecord
=
testDriver
.
readOutput
(
"${service.settings.outputTopic}-reporting"
,
...
...
@@ -107,22 +118,14 @@ class Tests {
StringDeserializer
()
)
}
val
processReport
=
testDriver
.
readOutput
(
service
.
settings
.
processReportTopic
,
StringDeserializer
(),
StringDeserializer
()
)
assertThat
(
processReport
)
.
isNotNull
.
hasFieldOrPropertyWithValue
(
"value"
,
params
.
processReport
)
}
private
fun
testParams
()
=
Stream
.
of
(
TestParams
(
"test1.yml"
,
"
key
1"
,
"
https://memobase.ch/instantiation/digital/BAZ-MEI_49885-
1"
,
"input1.nt"
,
"
key
"
,
"
https://memobase.ch/instantiation/digital/BAZ-MEI_49885-1
"
,
"output1.nt"
,
"output1.json"
,
"processOutput1.json"
...
...
src/test/resources/data/output1.json
0 → 100644
View file @
f7836b3c
{
"id"
:
"https://memobase.ch/instantiation/digital/BAZ-MEI_49885-1"
,
"status"
:
"SUCCESS"
,
"message"
:
"Added sftp locator link to digital resource."
}
\ No newline at end of file
src/test/resources/data/output1.nt
View file @
f7836b3c
<https://memobase.ch/instantiation/digital/BAZ-MEI_49885-1> <http://www.ebu.ch/metadata/ontologies/ebucore/ebucore#locator> "sftp://localhost:22000:/test_record_set_1/media/MEI_49885.jpg" .
<https://memobase.ch/instantiation/digital/BAZ-MEI_49885-1> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Instantiation> .
<https://memobase.ch/instantiation/digital/BAZ-MEI_49885-1> <https://www.ica.org/standards/RiC/ontology#identifiedBy> _:B .
<https://memobase.ch/instantiation/digital/BAZ-MEI_49885-1> <https://www.ica.org/standards/RiC/ontology#identifiedBy> _:B .
<https://memobase.ch/instantiation/digital/BAZ-MEI_49885-1> <https://www.ica.org/standards/RiC/ontology#instantiates> <https://memobase.ch/record/BAZ-MEI_49885> .
<https://memobase.ch/instantiation/digital/BAZ-MEI_49885-1> <https://www.ica.org/standards/RiC/ontology#isDerivedFromInstantiation> <https://memobase.ch/instantiation/physical/BAZ-MEI_49885-0> .
<https://memobase.ch/instantiation/digital/BAZ-MEI_49885-1> <https://www.ica.org/standards/RiC/ontology#regulatedBy> _:B .
<https://memobase.ch/instantiation/digital/BAZ-MEI_49885-1> <https://www.ica.org/standards/RiC/ontology#regulatedBy> _:B .
<https://memobase.ch/instantiation/digital/BAZ-MEI_49885-1> <https://www.ica.org/standards/RiC/ontology#type> "digitalObject" .
_:B <http://schema.org/sameAs> "http://rightsstatements.org/vocab/InC/1.0/" .
_:B <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Identifier> .
_:B <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Identifier> .
_:B <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Rule> .
_:B <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Rule> .
_:B <https://www.ica.org/standards/RiC/ontology#identifier> "MEI_49885" .
_:B <https://www.ica.org/standards/RiC/ontology#identifier> "https://memobase.ch/instantiation/digital/BAZ-MEI_49885-1" .
_:B <https://www.ica.org/standards/RiC/ontology#name> "In Copyright (InC)" .
_:B <https://www.ica.org/standards/RiC/ontology#name> "public" .
_:B <https://www.ica.org/standards/RiC/ontology#regulates> <https://memobase.ch/instantiation/digital/BAZ-MEI_49885-1> .
_:B <https://www.ica.org/standards/RiC/ontology#regulates> <https://memobase.ch/instantiation/digital/BAZ-MEI_49885-1> .
_:B <https://www.ica.org/standards/RiC/ontology#type> "access" .
_:B <https://www.ica.org/standards/RiC/ontology#type> "main" .
_:B <https://www.ica.org/standards/RiC/ontology#type> "original" .
_:B <https://www.ica.org/standards/RiC/ontology#type> "usage" .
\ No newline at end of file
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment