Commit 0ff0bfa4 authored by Jonas Waeber's avatar Jonas Waeber
Browse files

Fix json ld output

parent 3967501d
Pipeline #11793 passed with stages
in 4 minutes and 48 seconds
package org.memobase
import org.apache.jena.rdf.model.Model
import org.apache.jena.riot.RDFDataMgr
import org.apache.jena.riot.RDFFormat
import java.io.StringWriter
object Functions {
fun writeModel(model: Model): String {
val writer = StringWriter()
RDFDataMgr.write(writer, model, RDFFormat.JSONLD_COMPACT_FLAT)
return writer.toString().trim()
}
}
\ No newline at end of file
......@@ -74,7 +74,7 @@ class KafkaTopology(
.mapValues { value -> addEventType(value) }
.mapValues { value -> requestAdditionalRecordResources(value) }
.mapValues { value -> filterFedoraProperties(value) }
.mapValues { value -> writeModel(value) }
.mapValues { value -> Functions.writeModel(value) }
.to(settings.outputTopic + "-json-records")
objectBranches[1]
......@@ -82,7 +82,7 @@ class KafkaTopology(
.mapValues { value -> parseModel(value) }
.mapValues { value -> addEventType(value) }
.mapValues { value -> filterFedoraProperties(value) }
.mapValues { value -> writeModel(value) }
.mapValues { value -> Functions.writeModel(value) }
.to(settings.outputTopic + "-json-institutions")
objectBranches[2]
......@@ -90,7 +90,7 @@ class KafkaTopology(
.mapValues { value -> parseModel(value) }
.mapValues { value -> addEventType(value) }
.mapValues { value -> filterFedoraProperties(value) }
.mapValues { value -> writeModel(value) }
.mapValues { value -> Functions.writeModel(value) }
.to(settings.outputTopic + "-json-record-sets")
......@@ -134,6 +134,7 @@ class KafkaTopology(
* Downstream services should remove this property before publishing the data to the outside world!
*/
private fun addEventType(input: Pair<EventMessage, Model>): Model {
// TODO: Add filter for corporate body so that only memobase institutions are flagged.
listOf(RICO.Record, RICO.CorporateBody, RICO.RecordSet, RICO.Instantiation).forEach {
val record = input.second.listSubjectsWithProperty(RDF.type, it).toList()
if (record.isNotEmpty()) {
......@@ -199,10 +200,4 @@ class KafkaTopology(
}
return model
}
private fun writeModel(model: Model): String {
val writer = StringWriter()
RDFDataMgr.write(writer, model, Lang.NT)
return writer.toString().trim()
}
}
......@@ -17,6 +17,9 @@
*/
package org.memobase
import org.apache.jena.rdf.model.ModelFactory
import org.apache.jena.riot.Lang
import org.apache.jena.riot.RDFDataMgr
import java.io.File
import java.nio.charset.Charset
import java.util.stream.Stream
......@@ -26,9 +29,11 @@ import org.apache.kafka.streams.TopologyTestDriver
import org.apache.kafka.streams.test.ConsumerRecordFactory
import org.apache.logging.log4j.LogManager
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.TestInstance
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.MethodSource
import java.io.InputStream
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class Tests {
......@@ -38,8 +43,22 @@ class Tests {
private fun readFile(fileName: String): String {
return File("$resourcePath/$fileName").readText(Charset.defaultCharset())
}
private fun createInputStream(fileName: String): InputStream {
return File("$resourcePath/$fileName").inputStream()
}
private val regex = Regex("(_:B[A-Za-z0-9]+)")
@Test
fun `test json ld writer`() {
val model = ModelFactory.createDefaultModel()
RDFDataMgr.read(model, createInputStream("jsonld-test-input.nt"), Lang.NTRIPLES)
val result = Functions.writeModel(model)
assertThat(result)
.isEqualTo(readFile("jsonld-test-output.json"))
}
@ParameterizedTest
@MethodSource("kafkaTests")
fun `test kafka topology`(params: KafkaTestParams) {
......
This diff is collapsed.
{"@graph":[{"@id":"https://memobase.ch/instantiation/digital/SFW-CJS_0451-4_d-1","@type":"https://www.ica.org/standards/RiC/ontology#Instantiation","locator":"<![CDATA[https://media.zem.ch/01WS/1950/CJS_0451_d.mp4?start=249&end=320]]>","identifiedBy":"https://memobase.ch/instantiation/digital/SFW-CJS_0451-4_d-1#genidae28d239-ec0a-4b51-ac29-b4089bb78017","instantiates":"https://memobase.ch/record/SFW-CJS_0451-4_d","isDerivedFromInstantiation":"https://memobase.ch/instantiation/physical/SFW-CJS_0451-4_d-0","regulatedBy":["https://memobase.ch/instantiation/digital/SFW-CJS_0451-4_d-1#genid35c274da-3da2-4d0d-bddb-3e475c06e9e5","https://memobase.ch/instantiation/digital/SFW-CJS_0451-4_d-1#genid08606edb-a8b9-4193-82ff-fda34c73a109"],"type":"digitalObject"},{"@id":"https://memobase.ch/instantiation/digital/SFW-CJS_0451-4_d-1#genid08606edb-a8b9-4193-82ff-fda34c73a109","@type":"https://www.ica.org/standards/RiC/ontology#Rule","sameAs":"http://rightsstatements.org/vocab/InC/1.0/","name":"In Copyright (InC)","regulates":"https://memobase.ch/instantiation/digital/SFW-CJS_0451-4_d-1","type":"usage"},{"@id":"https://memobase.ch/instantiation/digital/SFW-CJS_0451-4_d-1#genid35c274da-3da2-4d0d-bddb-3e475c06e9e5","@type":"https://www.ica.org/standards/RiC/ontology#Rule","name":"public","regulates":"https://memobase.ch/instantiation/digital/SFW-CJS_0451-4_d-1","type":"access"},{"@id":"https://memobase.ch/instantiation/digital/SFW-CJS_0451-4_d-1#genidae28d239-ec0a-4b51-ac29-b4089bb78017","@type":"https://www.ica.org/standards/RiC/ontology#Identifier","identifier":"https://memobase.ch/instantiation/digital/SFW-CJS_0451-4_d-1","type":"main"},{"@id":"https://memobase.ch/instantiation/physical/SFW-CJS_0451-4_d-0","@type":"https://www.ica.org/standards/RiC/ontology#Instantiation","P60558":{"@language":"fr","@value":"NB"},"duration":"71","hasCarrierType":["https://memobase.ch/instantiation/physical/SFW-CJS_0451-4_d-0#genidb5627237-f3bc-4b43-b43c-0ac26d141225","https://memobase.ch/instantiation/physical/SFW-CJS_0451-4_d-0#genidd8795ebf-7180-400b-a597-9e1750a26830"],"hasDerivedInstantiation":"https://memobase.ch/instantiation/digital/SFW-CJS_0451-4_d-1","identifiedBy":["https://memobase.ch/instantiation/physical/SFW-CJS_0451-4_d-0#genid29d3abaf-2e6a-4b10-8760-b06efa845848","https://memobase.ch/instantiation/physical/SFW-CJS_0451-4_d-0#genid69e89c21-1c41-4189-bab9-2fe85f91fbaf"],"instantiates":"https://memobase.ch/record/SFW-CJS_0451-4_d","type":"physicalObject"},{"@id":"https://memobase.ch/instantiation/physical/SFW-CJS_0451-4_d-0#genid29d3abaf-2e6a-4b10-8760-b06efa845848","@type":"https://www.ica.org/standards/RiC/ontology#Identifier","identifier":"https://memobase.ch/instantiation/physical/SFW-CJS_0451-4_d-0","type":"main"},{"@id":"https://memobase.ch/instantiation/physical/SFW-CJS_0451-4_d-0#genid69e89c21-1c41-4189-bab9-2fe85f91fbaf","@type":"https://www.ica.org/standards/RiC/ontology#Identifier","identifier":"J2.143#1996/386#451-2#4*","type":"callNumber"},{"@id":"https://memobase.ch/instantiation/physical/SFW-CJS_0451-4_d-0#genidb5627237-f3bc-4b43-b43c-0ac26d141225","@type":"https://www.ica.org/standards/RiC/ontology#CarrierType","name":{"@language":"fr","@value":"nitrate de cellulose 35mm"}},{"@id":"https://memobase.ch/instantiation/physical/SFW-CJS_0451-4_d-0#genidd8795ebf-7180-400b-a597-9e1750a26830","@type":"https://www.ica.org/standards/RiC/ontology#CarrierType","sameAs":"http://www.wikidata.org/entity/Q226528","name":{"@language":"de","@value":"35 mm Film"},"resultsFrom":"https://memobase.ch/instantiation/physical/SFW-CJS_0451-4_d-0#genid374b5e19-fae7-4d53-aad2-e7fb726244d4"},{"@id":"https://memobase.ch/record/SFW-CJS_0451-4_d","@type":"https://www.ica.org/standards/RiC/ontology#Record","eventType":"Create","abstract":{"@language":"fr","@value":"Communiqué : <br> Pendant la Semaine Suisse un grand magasin de Zurich a consacré à l’aviation une exposition particulièrement bien documentée. <br> <br> Commentaire : <br> Pendant la Semaine Suisse, dans un grand magasin de Zurich une exposition aéronautique évoque les pionniers de notre aviation : le célèbre zinc de Dufaux, / un mémorable décollage de Taddeoli. / Ailleurs nos pilotes militaires, leurs avions et leur premier chef, / Oscar Bider, héros de la jeunesse, comme, plus tard, Walter Mittelholzer. / Une autre œuvre de pionnier : l’avion sans queue de Soldenhoff, le premier appareil de ce genre qui soit utilisable. / Le vol à voile est aussi représenté de façon captivante : un modèle sert à la démonstration des courants thermiques ascendants. / Rien ne manque dans cette exposition pas même une soufflerie. / Rarement tests aérodynamiques furent aussi éloquents. / Notre compagnie nationale d’aviation expose une maquette de sa plus récente acquisition, faite avec l’aide de la Confédération, et voici Kloten terminé, du moins en miniature. / <br> <br> <![CDATA[<a href=\"https://media.zem.ch/01WS/1950/Communique_0451.pdf\" target=\"_blank\">Communiqué_0451.pdf</a>]]>"},"issued":"https://memobase.ch/record/SFW-CJS_0451-4_d#genidf2467703-18d4-40e5-8a31-19030f516e61","spatial":"https://memobase.ch/record/SFW-CJS_0451-4_d#genidbcbfa2f2-4b66-4490-b762-7cf62d1218e9","P60441":"https://memobase.ch/record/SFW-CJS_0451-4_d#genidc7e195f2-1a98-43ae-8527-9e27334d86ba","P60451":"https://memobase.ch/institution/Memoriav","P60556":"https://memobase.ch/record/SFW-CJS_0451-4_d#genidddc2be0d-92c0-44cd-bdc2-864068c95e90","sameAs":"https://www.recherche.bar.admin.ch/recherche/#/fr/recherche/plan-d-archivage/30689062","hasGenre":"https://memobase.ch/record/SFW-CJS_0451-4_d#genid981baf6b-9e1c-4e93-bfec-300d573acad2","descriptiveNote":{"@language":"fr","@value":"Source à mentionner dans des citations et publications : <br> Archives fédérales suisses, J2.143#1996/386#451-2#4*, Ciné-Journal suisse du 03.11.1950, Les ailes (0451-4)"},"hasInstantiation":["https://memobase.ch/instantiation/digital/SFW-CJS_0451-4_d-1","https://memobase.ch/instantiation/physical/SFW-CJS_0451-4_d-0"],"hasLanguage":["https://memobase.ch/record/SFW-CJS_0451-4_d#genid728861b6-7528-45f3-945a-f3cb6f574ba6","https://memobase.ch/record/SFW-CJS_0451-4_d#genidb7b12538-3c01-4075-a396-00b06e776031"],"hasSubject":["https://memobase.ch/record/SFW-CJS_0451-4_d#genidd3288cd2-4f82-4e80-b3d0-19b1f79356ef","https://memobase.ch/record/SFW-CJS_0451-4_d#genid6c189fbc-89c2-4692-bd08-425524cfd84a","https://memobase.ch/record/SFW-CJS_0451-4_d#genidaf8153fe-5058-493b-a00e-49d063fa523e"],"hasTitle":["https://memobase.ch/record/SFW-CJS_0451-4_d#genida3b0db1c-fd49-4e1d-bc64-827029a23cbe","https://memobase.ch/record/SFW-CJS_0451-4_d#genid8c3fdb42-7314-4c92-af10-c3bbbe4daa67","https://memobase.ch/record/SFW-CJS_0451-4_d#genid5f9d7230-2866-4c98-9734-27097653e494"],"heldBy":"https://memobase.ch/institution/SFW","identifiedBy":["https://memobase.ch/record/SFW-CJS_0451-4_d#genidb5b2f310-faa8-41ef-937f-e2677c9fb8ea","https://memobase.ch/record/SFW-CJS_0451-4_d#genid85f8450d-2485-4a0b-b3c3-e68422973675"],"isPartOf":"https://memobase.ch/recordSet/SFW_CJS_CGS","publishedBy":"https://memobase.ch/record/SFW-CJS_0451-4_d#genidb9f4106e-b69e-4358-8fa9-0ac047b64404","recordResourceOrInstantiationIsSourceOfCreationRelation":"https://memobase.ch/record/SFW-CJS_0451-4_d#genidaae45df3-1e3b-48a1-86b7-da6f78a4cfd3","regulatedBy":"https://memobase.ch/record/SFW-CJS_0451-4_d#genid5ca8e746-38e5-4065-bd8b-a7b1a80164c6","title":{"@language":"fr","@value":"Les ailes (0451-4)"},"type":"Film"},{"@id":"https://memobase.ch/record/SFW-CJS_0451-4_d#genid3e5572fe-c210-4041-b848-0c5a040438fc","@type":"https://www.ica.org/standards/RiC/ontology#CorporateBody","agentIsTargetOfCreationRelation":"https://memobase.ch/record/SFW-CJS_0451-4_d#genidaae45df3-1e3b-48a1-86b7-da6f78a4cfd3","name":{"@language":"fr","@value":"Ciné-Journal suisse (CJS)"}},{"@id":"https://memobase.ch/record/SFW-CJS_0451-4_d#genid5ca8e746-38e5-4065-bd8b-a7b1a80164c6","@type":"https://www.ica.org/standards/RiC/ontology#Rule","name":"Cinémathèque suisse, Archives fédérales suisses","regulates":"https://memobase.ch/record/SFW-CJS_0451-4_d","type":"holder"},{"@id":"https://memobase.ch/record/SFW-CJS_0451-4_d#genid5f9d7230-2866-4c98-9734-27097653e494","@type":"https://www.ica.org/standards/RiC/ontology#Title","title":{"@language":"fr","@value":"Les ailes (0451-4)"},"type":"main"},{"@id":"https://memobase.ch/record/SFW-CJS_0451-4_d#genid6c189fbc-89c2-4692-bd08-425524cfd84a","@type":"http://www.w3.org/2004/02/skos/core#Concept","editorialNote":"Ursprungsfeld: Deskriptoren","prefLabel":{"@language":"fr","@value":"CULTURE, EXPOSITION"}},{"@id":"https://memobase.ch/record/SFW-CJS_0451-4_d#genid728861b6-7528-45f3-945a-f3cb6f574ba6","@type":"https://www.ica.org/standards/RiC/ontology#Language","name":{"@language":"fr","@value":"français"},"type":"caption"},{"@id":"https://memobase.ch/record/SFW-CJS_0451-4_d#genid85f8450d-2485-4a0b-b3c3-e68422973675","@type":"https://www.ica.org/standards/RiC/ontology#Identifier","identifier":"CJS_0451-4_d","type":"original"},{"@id":"https://memobase.ch/record/SFW-CJS_0451-4_d#genid8c3fdb42-7314-4c92-af10-c3bbbe4daa67","@type":"https://www.ica.org/standards/RiC/ontology#Title","title":{"@language":"fr","@value":"Ciné-Journal suisse"},"type":"series"},{"@id":"https://memobase.ch/record/SFW-CJS_0451-4_d#genid981baf6b-9e1c-4e93-bfec-300d573acad2","@type":"http://www.w3.org/2004/02/skos/core#Concept","editorialNote":"Ursprungsfeld: Genre","prefLabel":{"@language":"fr","@value":"Cinéjournal"}},{"@id":"https://memobase.ch/record/SFW-CJS_0451-4_d#genida3b0db1c-fd49-4e1d-bc64-827029a23cbe","@type":"https://www.ica.org/standards/RiC/ontology#Title","title":{"@language":"fr","@value":"Ciné-Journal suisse du 03.11.1950"},"type":"broadcast"},{"@id":"https://memobase.ch/record/SFW-CJS_0451-4_d#genidaae45df3-1e3b-48a1-86b7-da6f78a4cfd3","@type":"https://www.ica.org/standards/RiC/ontology#CreationRelation","creationRelationHasSource":"https://memobase.ch/record/SFW-CJS_0451-4_d","creationRelationHasTarget":"https://memobase.ch/record/SFW-CJS_0451-4_d#genid3e5572fe-c210-4041-b848-0c5a040438fc","name":"Autor","type":"creator"},{"@id":"https://memobase.ch/record/SFW-CJS_0451-4_d#genidaf8153fe-5058-493b-a00e-49d063fa523e","@type":"https://www.ica.org/standards/RiC/ontology#Person","name":{"@language":"fr","@value":"Soldenhoff; Emile Taddeoli; Oskar Bider; Armand Dufaux; Walter Mittelholzer"}},{"@id":"https://memobase.ch/record/SFW-CJS_0451-4_d#genidb5b2f310-faa8-41ef-937f-e2677c9fb8ea","@type":"https://www.ica.org/standards/RiC/ontology#Identifier","identifier":"https://memobase.ch/record/SFW-CJS_0451-4_d","type":"main"},{"@id":"https://memobase.ch/record/SFW-CJS_0451-4_d#genidb7b12538-3c01-4075-a396-00b06e776031","@type":"https://www.ica.org/standards/RiC/ontology#Language","name":{"@language":"fr","@value":"allemand"},"type":"content"},{"@id":"https://memobase.ch/record/SFW-CJS_0451-4_d#genidb9f4106e-b69e-4358-8fa9-0ac047b64404","@type":"https://www.ica.org/standards/RiC/ontology#CorporateBody","eventType":"Create","name":{"@language":"fr","@value":"CJS"}},{"@id":"https://memobase.ch/record/SFW-CJS_0451-4_d#genidbcbfa2f2-4b66-4490-b762-7cf62d1218e9","@type":"https://www.ica.org/standards/RiC/ontology#Place","name":{"@language":"fr","@value":"Europe occidentale"}},{"@id":"https://memobase.ch/record/SFW-CJS_0451-4_d#genidc7e195f2-1a98-43ae-8527-9e27334d86ba","@type":"https://www.ica.org/standards/RiC/ontology#CorporateBody","name":{"@language":"fr","@value":"CJS"}},{"@id":"https://memobase.ch/record/SFW-CJS_0451-4_d#genidd3288cd2-4f82-4e80-b3d0-19b1f79356ef","@type":"http://www.w3.org/2004/02/skos/core#Concept","editorialNote":"Ursprungsfeld: Deskriptoren","prefLabel":{"@language":"fr","@value":"TRANSPORT, AVION"}},{"@id":"https://memobase.ch/record/SFW-CJS_0451-4_d#genidddc2be0d-92c0-44cd-bdc2-864068c95e90","@type":"https://www.ica.org/standards/RiC/ontology#Place","name":{"@language":"fr","@value":"Zürich, ZH (Schweiz)"}},{"@id":"https://memobase.ch/record/SFW-CJS_0451-4_d#genidf2467703-18d4-40e5-8a31-19030f516e61","@type":"https://www.ica.org/standards/RiC/ontology#DateRange","expressedDate":"18570"}],"@context":{"name":{"@id":"https://www.ica.org/standards/RiC/ontology#name"},"type":{"@id":"https://www.ica.org/standards/RiC/ontology#type"},"regulates":{"@id":"https://www.ica.org/standards/RiC/ontology#regulates","@type":"@id"},"sameAs":{"@id":"http://schema.org/sameAs"},"resultsFrom":{"@id":"https://www.ica.org/standards/RiC/ontology#resultsFrom","@type":"@id"},"hasInstantiation":{"@id":"https://www.ica.org/standards/RiC/ontology#hasInstantiation","@type":"@id"},"hasTitle":{"@id":"https://www.ica.org/standards/RiC/ontology#hasTitle","@type":"@id"},"issued":{"@id":"http://purl.org/dc/terms/issued","@type":"@id"},"title":{"@id":"https://www.ica.org/standards/RiC/ontology#title"},"hasLanguage":{"@id":"https://www.ica.org/standards/RiC/ontology#hasLanguage","@type":"@id"},"descriptiveNote":{"@id":"https://www.ica.org/standards/RiC/ontology#descriptiveNote"},"hasGenre":{"@id":"http://www.ebu.ch/metadata/ontologies/ebucore/ebucore#hasGenre","@type":"@id"},"P60441":{"@id":"http://rdaregistry.info/Elements/u/P60441","@type":"@id"},"heldBy":{"@id":"https://www.ica.org/standards/RiC/ontology#heldBy","@type":"@id"},"hasSubject":{"@id":"https://www.ica.org/standards/RiC/ontology#hasSubject","@type":"@id"},"P60451":{"@id":"http://rdaregistry.info/Elements/u/P60451","@type":"@id"},"publishedBy":{"@id":"https://www.ica.org/standards/RiC/ontology#publishedBy","@type":"@id"},"eventType":{"@id":"http://memobase.ch/internal/eventType"},"isPartOf":{"@id":"https://www.ica.org/standards/RiC/ontology#isPartOf","@type":"@id"},"recordResourceOrInstantiationIsSourceOfCreationRelation":{"@id":"https://www.ica.org/standards/RiC/ontology#recordResourceOrInstantiationIsSourceOfCreationRelation","@type":"@id"},"regulatedBy":{"@id":"https://www.ica.org/standards/RiC/ontology#regulatedBy","@type":"@id"},"abstract":{"@id":"http://purl.org/dc/terms/abstract"},"spatial":{"@id":"http://purl.org/dc/terms/spatial","@type":"@id"},"P60556":{"@id":"http://rdaregistry.info/Elements/u/P60556","@type":"@id"},"identifiedBy":{"@id":"https://www.ica.org/standards/RiC/ontology#identifiedBy","@type":"@id"},"agentIsTargetOfCreationRelation":{"@id":"https://www.ica.org/standards/RiC/ontology#agentIsTargetOfCreationRelation","@type":"@id"},"prefLabel":{"@id":"http://www.w3.org/2004/02/skos/core#prefLabel"},"editorialNote":{"@id":"http://www.w3.org/2004/02/skos/core#editorialNote"},"identifier":{"@id":"https://www.ica.org/standards/RiC/ontology#identifier"},"expressedDate":{"@id":"https://www.ica.org/standards/RiC/ontology#expressedDate"},"creationRelationHasSource":{"@id":"https://www.ica.org/standards/RiC/ontology#creationRelationHasSource","@type":"@id"},"creationRelationHasTarget":{"@id":"https://www.ica.org/standards/RiC/ontology#creationRelationHasTarget","@type":"@id"},"locator":{"@id":"http://www.ebu.ch/metadata/ontologies/ebucore/ebucore#locator"},"instantiates":{"@id":"https://www.ica.org/standards/RiC/ontology#instantiates","@type":"@id"},"isDerivedFromInstantiation":{"@id":"https://www.ica.org/standards/RiC/ontology#isDerivedFromInstantiation","@type":"@id"},"hasCarrierType":{"@id":"https://www.ica.org/standards/RiC/ontology#hasCarrierType","@type":"@id"},"P60558":{"@id":"http://rdaregistry.info/Elements/u/P60558"},"duration":{"@id":"http://www.ebu.ch/metadata/ontologies/ebucore/ebucore#duration"},"hasDerivedInstantiation":{"@id":"https://www.ica.org/standards/RiC/ontology#hasDerivedInstantiation","@type":"@id"}}}
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment