Commit dbf98541 authored by Jonas Waeber's avatar Jonas Waeber
Browse files

Add reporting & fix tests

parent ad31af92
Pipeline #25293 passed with stages
in 4 minutes and 13 seconds
package org.memobase
import ch.memobase.reporting.Report
import ch.memobase.reporting.ReportStatus
import com.beust.klaxon.Klaxon
import com.beust.klaxon.KlaxonException
import java.io.StringReader
......@@ -10,20 +12,20 @@ object JSON {
private val log = LogManager.getLogger("JsonParser")
private val klaxon = Klaxon()
fun parseJson(key: String, data: String): List<Input> {
fun parseJson(key: String, data: String): Pair<List<Input>, Report> {
return try {
val int = klaxon.parse<Input>(StringReader(data))
return if (int == null) {
log.error("Could not parse input in message with key: $key.")
emptyList()
Pair(emptyList(), Report(key, ReportStatus.fatal, "Nothing to parse for item $key.", Util.step))
}
else listOf(int)
else Pair(listOf(int), Report(key, ReportStatus.success, "", Util.step))
} catch (ex: KlaxonException) {
log.error(ex.localizedMessage)
emptyList()
Pair(emptyList(), Report(key, ReportStatus.fatal, "JSON Parse Error: ${ex.localizedMessage}.", Util.step))
} catch (ex: ClassCastException) {
log.error(ex.localizedMessage)
emptyList()
Pair(emptyList(), Report(key, ReportStatus.fatal, "JSON Parse Error: ${ex.localizedMessage}.", Util.step))
}
}
}
\ No newline at end of file
......@@ -18,6 +18,8 @@
package org.memobase
import ch.memobase.reporting.Report
import ch.memobase.reporting.ReportStatus
import ch.memobase.settings.SettingsLoader
import org.apache.jena.rdf.model.Model
import org.apache.kafka.streams.StreamsBuilder
......@@ -33,23 +35,69 @@ class KafkaTopology(private val settings: SettingsLoader) {
fun build(): Topology {
val builder = StreamsBuilder()
val stream = builder.stream<String, String>(settings.inputTopic)
stream
.flatMapValues { value -> JSON.parseJson("", value) }
.flatMapValues { value -> mapJson(value) }
val parsedJson = stream
.mapValues { value -> JSON.parseJson("", value) }
parsedJson
.filter { _, value -> value.second.status == ReportStatus.fatal }
.mapValues { value -> value.second.toJson() }
.to(settings.processReportTopic)
val parsedRdfObject = parsedJson.flatMapValues { value -> value.first }
.mapValues { readOnlyKey, value -> mapJson(readOnlyKey, value) }
parsedRdfObject
.filter { _, value -> value.second.status == ReportStatus.fatal }
.mapValues { value -> value.second.toJson() }
.to(settings.processReportTopic)
val successfulWrite = parsedRdfObject.flatMapValues { value -> value.first }
.map { _, value -> Util.writeModel(value.first, value.second) }
successfulWrite
.to(settings.outputTopic)
successfulWrite.mapValues { key, _ -> Report(key, ReportStatus.success, "", Util.step).toJson() }
.to(settings.processReportTopic)
return builder.build()
}
private fun mapJson(input: Input): List<Pair<String, Model>> {
return when (input) {
is Institution -> listOf(transformer.createInstitution(input))
is RecordSet -> listOf(transformer.createRecordSet(input))
else -> {
log.error("Could not cast input to institution or record set: $input.")
return emptyList()
private fun mapJson(key: String, input: Input): Pair<List<Pair<String, Model>>, Report> {
return try {
when (input) {
is Institution -> Pair(
listOf(transformer.createInstitution(input)),
Report(key, ReportStatus.success, "", Util.step)
)
is RecordSet -> Pair(
listOf(transformer.createRecordSet(input)),
Report(key, ReportStatus.success, "", Util.step)
)
else -> {
log.error("Could not cast input to institution or record set: $input.")
return Pair(
emptyList(),
Report(
key,
ReportStatus.fatal,
"Input is not a valid institution or record set (type: ${input.type}).",
Util.step
)
)
}
}
} catch (ex: Exception) {
log.error(ex.localizedMessage)
return Pair(
emptyList(),
Report(
key,
ReportStatus.fatal,
"Exception ${ex.javaClass}: ${ex.localizedMessage}.",
Util.step
)
)
}
}
}
......@@ -38,6 +38,7 @@ object Util {
const val drupalInstitutionTypeIdentifier = "node--institution"
const val drupalRecordSetTypeIdentifier = "node--record_set"
const val step = "gi-drupal-syncer"
/**
* This constant is used to recognize if the related record set is a record set indexed in Memobase.
......@@ -76,7 +77,7 @@ object Util {
fun writeModel(uri: String, model: Model): KeyValue<String, String> {
return StringWriter().use { writer ->
RDFDataMgr.write(writer, model, RDFFormat.NTRIPLES_UTF8)
return@use KeyValue(uri, writer.toString().trim())
KeyValue(uri, writer.toString().trim())
}
}
}
\ No newline at end of file
......@@ -69,7 +69,7 @@ class TestInstitutions {
)
testDriver.pipeInput(
factory.create(
service.settings.inputTopic, null, readFile("completeExample.json")
service.settings.inputTopic, "completeExample", readFile("completeExample.json")
)
)
......
......@@ -66,7 +66,7 @@ class TestRecordSets {
@Test
fun `test record set transform`() {
val service = Service("test1.yml")
val input = JSON.parseJson("testComplete", readFile("completeExample.json"))[0]
val input = JSON.parseJson("testComplete", readFile("completeExample.json")).first[0]
val result = RdfTransformer(service.settings.appSettings).createRecordSet(input as RecordSet)
assertAll("",
{
......
......@@ -53,14 +53,14 @@ mbcb:completeInstitution
wdt:P669 "Street" ;
wdt:P670 "Address"
] ;
rico:identifiedBy [ a rico:Identifier ;
rico:identifier "OLD_MEMOBASE_ID" ;
rico:type "oldMemobase"
] ;
rico:identifiedBy [ a rico:Identifier ;
rico:identifier "completeInstitution" ;
rico:type "main"
] ;
rico:identifiedBy [ a rico:Identifier ;
rico:identifier "OLD_MEMOBASE_ID" ;
rico:type "oldMemobase"
] ;
rico:isHolderOf "https://memobase.ch/recordSet/testComplete" ;
rico:name "Complete Institution (FR)"@fr , "Complete Institution (IT)"@it , "Complete Institution (DE)"@de ;
rico:type "memobaseInstitution" .
......@@ -32,13 +32,13 @@ mbrs:testComplete a rico:RecordSet ;
rico:type "memoriavProject"
] ;
rdau:P60451 [ a rico:CorporateBody ;
schema:sameAs "http://example.org" ;
rico:title "Projektname (IT)"@it ;
schema:sameAs "http://memobase.ch/de/stock/detail/Tanzarchiv-121" ;
rico:title "Projektname (DE)"@de ;
rico:type "memoriavProject"
] ;
rdau:P60451 [ a rico:CorporateBody ;
schema:sameAs "http://memobase.ch/de/stock/detail/Tanzarchiv-121" ;
rico:title "Projektname (DE)"@de ;
schema:sameAs "http://example.org" ;
rico:title "Projektname (IT)"@it ;
rico:type "memoriavProject"
] ;
rdau:P60496 "<p>Zugang Memobase (DE)</p>\r\n\r\n<p>Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>"@de , "<p>Zugang Memobase (FR)</p>\r\n\r\n<p>Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>"@fr , "<p>Zugang Memobase (IT)</p>\r\n\r\n<p>Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>"@it ;
......@@ -58,14 +58,14 @@ mbrs:testComplete a rico:RecordSet ;
schema:sameAs "https://example.org"@de , "https://bestand.org"@fr , "https://example.org"@it ;
rico:title "Original Bestand (DE)"@de , "Original Bestand (FR)"@fr , "Original Bestand (IT)"@it
] ;
rico:hasTitle [ a rico:Title ;
rico:title "Complete Record Set (DE)"@de , "Complete Record Set (FR)"@fr , "Complete Record Set (IT)"@it ;
rico:type "main"
] ;
rico:hasTitle [ a rico:Title ;
rico:title "Originaltitle des Bestandes (DE)"@de , "Originaltitel des Bestandes"@fr , "Originaltitel des Bestandes (IT)"@it ;
rico:type "original"
] ;
rico:hasTitle [ a rico:Title ;
rico:title "Complete Record Set (DE)"@de , "Complete Record Set (FR)"@fr , "Complete Record Set (IT)"@it ;
rico:type "main"
] ;
rico:heldBy "https://memobase.ch/institution/completeInstitution" ;
rico:history "<p>Kontext (DE)</p>\r\n\r\n<p>Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>"@de , "<p>Kontext (IT)</p>\r\n\r\n<p>Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>"@it , "<p>Kontext (FR)</p>\r\n\r\n<p>Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>"@fr ;
rico:identifiedBy [ a rico:Identifier ;
......@@ -76,14 +76,14 @@ mbrs:testComplete a rico:RecordSet ;
rico:identifier "ORIGINAL_ID" ;
rico:type "original"
] ;
rico:identifiedBy [ a rico:Identifier ;
rico:identifier "OLD_MEMOBASE_ID" ;
rico:type "oldMemobase"
] ;
rico:identifiedBy [ a rico:Identifier ;
rico:identifier "Original-Signatur des Bestandes" ;
rico:type "callNumber"
] ;
rico:identifiedBy [ a rico:Identifier ;
rico:identifier "OLD_MEMOBASE_ID" ;
rico:type "oldMemobase"
] ;
rico:integrity "<div class=\"page-generator__output js-generator-output\">Auswahl (FR)</div>\r\n\r\n<div class=\"page-generator__output js-generator-output\">Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</div>"@fr , "<p>Auswahl / Vollständigkeit (DE)</p>\r\n\r\n<p>Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>"@de , "<div class=\"page-generator__output js-generator-output\">Auswahl/Vollständigkeit (IT)</div>\r\n\r\n<div class=\"page-generator__output js-generator-output\">Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</div>"@it ;
rico:isAssociatedWithDate [ a rico:DateRange ;
rico:normalizedDateValue "1920/2020"
......@@ -91,9 +91,9 @@ mbrs:testComplete a rico:RecordSet ;
rico:isRecordResourceAssociatedWithRecordResource
mbrs:testComplete ;
rico:isRecordResourceAssociatedWithRecordResource
[ a rico:Record ;
schema:sameAs "https://example.org/" ;
rico:title "Dokument (IT)"@it ;
[ a rico:RecordSet ;
schema:sameAs "https://example.org" ;
rico:title "Verwandter Bestand Extern (DE)"@de ;
rico:type "related"
] ;
rico:isRecordResourceAssociatedWithRecordResource
......@@ -105,13 +105,13 @@ mbrs:testComplete a rico:RecordSet ;
rico:isRecordResourceAssociatedWithRecordResource
[ a rico:Record ;
schema:sameAs "https://example.org/" ;
rico:title "Dokumente (FR)"@fr ;
rico:title "Dokument (IT)"@it ;
rico:type "related"
] ;
rico:isRecordResourceAssociatedWithRecordResource
[ a rico:RecordSet ;
schema:sameAs "https://example.org" ;
rico:title "Verwandter Bestand Extern (DE)"@de ;
[ a rico:Record ;
schema:sameAs "https://example.org/" ;
rico:title "Dokumente (FR)"@fr ;
rico:type "related"
] ;
rico:isRecordResourceAssociatedWithRecordResource
......@@ -128,7 +128,7 @@ mbrs:testComplete a rico:RecordSet ;
] ;
rico:isSubjectOf [ a rico:Record ;
schema:sameAs "https://example.org/" ;
rico:title "Publikation (FR)"@fr ;
rico:title "Publikation (DE)"@de ;
rico:type "publication"
] ;
rico:isSubjectOf [ a rico:Record ;
......@@ -138,10 +138,10 @@ mbrs:testComplete a rico:RecordSet ;
] ;
rico:isSubjectOf [ a rico:Record ;
schema:sameAs "https://example.org/" ;
rico:title "Publikation (DE)"@de ;
rico:title "Publikation (FR)"@fr ;
rico:type "publication"
] ;
rico:modificationDate "2021-04-15T12:00:55.505"^^xsd:dateTime ;
rico:modificationDate "2021-04-22T11:23:55.133"^^xsd:dateTime ;
rico:publicationDate "2020-09-04"^^xsd:date ;
rico:recordResourceExtent "<p>Umfang (IT)</p>\r\n\r\n<p>Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>"@it , "<p>Umfang (FR)</p>\r\n\r\n<p>Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>"@fr , "<p>Umfang (DE)</p>\r\n\r\n<p>Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>"@de ;
rico:recordResourceOrInstantiationIsTargetOfRecordResourceHoldingRelation
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment