Commit f57f06fe authored by Jonas Waeber's avatar Jonas Waeber
Browse files

Refactor mapper service

- Update dependency
-
parent 45b0a892
......@@ -35,7 +35,7 @@ dependencies {
// https://mvnrepository.com/artifact/org.elasticsearch.client/elasticsearch-rest-high-level-client
//compile group: 'org.elasticsearch.client', name: 'elasticsearch-rest-high-level-client', version: '7.1.0'
implementation 'org.memobase:memobase-service-utilities:1.5.0'
implementation 'org.memobase:memobase-service-utilities:1.11.0'
// Logging Framework
implementation "org.apache.logging.log4j:log4j-api:${log4jV}"
......
......@@ -18,33 +18,15 @@
package org.memobase
import org.apache.kafka.streams.KafkaStreams
import org.apache.logging.log4j.LogManager
import org.memobase.settings.SettingsLoader
import kotlin.system.exitProcess
class App {
companion object {
private val log = LogManager.getLogger("MapperService")
private val log = LogManager.getLogger("MapperServiceApp")
@JvmStatic fun main(args: Array<String>) {
try {
val settings = SettingsLoader(
listOf(
"institutionId",
"recordSetId",
"configs"
),
useStreamsConfig = true
)
val topology = KafkaTopology(settings).build()
val stream = KafkaStreams(topology, settings.kafkaStreamsSettings)
stream.use {
it.start()
while (stream.state().isRunning) {
Thread.sleep(10_000L)
}
}
Service().run()
} catch (ex: Exception) {
ex.printStackTrace()
log.error("Stopping application due to error: " + ex.message)
......
......@@ -80,18 +80,6 @@ class KafkaTopology(
val recordStream = completedMappingStream.mapValues { value -> value.writeRecord() }
objectOutput(recordStream)
/*
val physicalObjectStream = completedMappingStream
.filter { _, value -> value.hasPhysicalObject() }
.mapValues { value -> value.writePhysicalObject() }
val digitalObjectStream = completedMappingStream
.filter { _, value -> value.hasDigitalObject() }
.mapValues { value -> value.writeDigitalObject() }
objectOutput(physicalObjectStream)
objectOutput(digitalObjectStream)
*/
return builder.build()
}
......
......@@ -25,9 +25,7 @@ data class Report(
val status: String,
val message: String
) {
fun toJson(): String {
return Klaxon().toJsonString(this)
}
}
......@@ -16,29 +16,38 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.memobase.rdf
package org.memobase
import org.apache.jena.rdf.model.Property
import org.apache.jena.rdf.model.Resource
import org.apache.jena.rdf.model.ResourceFactory
import org.apache.kafka.streams.KafkaStreams
import org.apache.logging.log4j.LogManager
import org.memobase.settings.SettingsLoader
object EBUCORE {
class Service(file: String = "app.yml") {
private val log = LogManager.getLogger("TableDataService")
val hasGenre = prop("hasGenre")
val hasFormat = prop("hasFormat")
val hasMedium = prop("hasMedium")
val duration = prop("duration")
val displayAspectRatio = prop("displayAspectRatio")
val audioTrackConfiguration = prop("audioTrackConfiguration")
val playbackSpeed = prop("playbackSpeed")
val hasStandard = prop("hasStandard")
val locator = prop("locator")
val settings = SettingsLoader(
listOf(
"institutionId",
"institutionIdOld",
"recordSetId",
"recordSetIdOld",
"configs"
),
file,
useStreamsConfig = true,
readSftpSettings = true
)
private fun prop(name: String): Property {
return ResourceFactory.createProperty(NS.ebucore, name)
}
val topology = KafkaTopology(settings).build()
private val stream = KafkaStreams(topology, settings.kafkaStreamsSettings)
private fun res(name: String): Resource {
return ResourceFactory.createResource(NS.ebucore + name)
fun run() {
stream.use {
it.start()
while (stream.state().isRunning) {
log.info("Service is running.")
Thread.sleep(10_000L)
}
}
}
}
\ No newline at end of file
......@@ -25,7 +25,7 @@ import org.memobase.rdf.NS
import org.memobase.rdf.RICO
class DigitalObject(id: String, institutionId: String, count: Int) : Instantiation(institutionId) {
override val resource: Resource = model.createResource(NS.memdo + institutionId + "-" + StringHelpers.normalizeId(id) + "-" + count)
override val resource: Resource = model.createResource(NS.mbdo + institutionId + "-" + StringHelpers.normalizeId(id) + "-" + count)
init {
addRdfType(RICO.Instantiation)
resource.addProperty(RICO.type, "digitalObject")
......
......@@ -25,7 +25,7 @@ import org.memobase.rdf.NS
import org.memobase.rdf.RICO
class PhysicalObject(id: String, institutionId: String, count: Int) : Instantiation(institutionId) {
override val resource: Resource = model.createResource(NS.mempo + institutionId + "-" + StringHelpers.normalizeId(id) + "-" + count)
override val resource: Resource = model.createResource(NS.mbpo + institutionId + "-" + StringHelpers.normalizeId(id) + "-" + count)
init {
addRdfType(RICO.Instantiation)
resource.addProperty(RICO.type, "physicalObject")
......
......@@ -28,7 +28,7 @@ import org.memobase.rdf.RICO
class Record(id: String, type: String, recordSetId: String, institutionId: String, hasSponsoringAgent: Boolean) :
RecordResource(institutionId) {
override val resource: Resource = model.createResource(NS.memr + institutionId + "-" + StringHelpers.normalizeId(id))
override val resource: Resource = model.createResource(NS.mbr + institutionId + "-" + StringHelpers.normalizeId(id))
init {
addRdfType(RICO.Record)
resource.addProperty(RICO.type, type)
......@@ -44,5 +44,5 @@ class Record(id: String, type: String, recordSetId: String, institutionId: Strin
resource.addProperty(RICO.hasInstantiation, instantiation.resource)
}
private fun recordSetUri(id: String): Resource = model.createResource(uri(NS.memrs, id))
private fun recordSetUri(id: String): Resource = model.createResource(uri(NS.mbrs, id))
}
\ No newline at end of file
......@@ -44,7 +44,7 @@ abstract class RecordResource(institutionId: String) : IResource {
init {
model.setNsPrefix("rico", NS.rico)
model.setNsPrefix("rdf", NS.rdf)
model.setNsPrefix("dct", NS.dct)
model.setNsPrefix("dct", NS.dcterms)
model.setNsPrefix("ebucore", NS.ebucore)
model.setNsPrefix("skos", NS.skos)
model.setNsPrefix("rdau", NS.rdau)
......@@ -174,5 +174,5 @@ abstract class RecordResource(institutionId: String) : IResource {
protected fun uri(ns: String, name: String): String = ns + name
private fun institutionUri(id: String): Resource = model.createResource(uri(NS.memint, id))
private fun institutionUri(id: String): Resource = model.createResource(uri(NS.mbcb, id))
}
\ No newline at end of file
......@@ -27,7 +27,7 @@ import org.memobase.rdf.SCHEMA
object KEYS {
val memoriavUri = NS.memint + "Memoriav"
const val memoriavUri = NS.mbcb + "Memoriav"
val validRecordTypeValues = listOf("Film", "Foto", "Radio", "Ton", "Tonbildschau", "TV", "Video")
......
/*
* mapper-service
* Copyright (C) 2020 Memoriav
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.memobase.rdf
import org.apache.jena.rdf.model.Property
import org.apache.jena.rdf.model.Resource
import org.apache.jena.rdf.model.ResourceFactory
object DC {
val abstract: Property = prop("abstract")
val created: Property = prop("created")
val issued: Property = prop("issued")
val temporal: Property = prop("temporal")
val spatial: Property = prop("spatial")
val relation: Property = prop("relation")
private fun prop(name: String): Property {
return ResourceFactory.createProperty(NS.dct, name)
}
private fun res(name: String): Resource {
return ResourceFactory.createResource(NS.dct + name)
}
}
\ No newline at end of file
/*
* mapper-service
* Copyright (C) 2020 Memoriav
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.memobase.rdf
object NS {
val rdf = "http://www.w3.org/1999/02/22-rdf-syntax-ns#"
val rdfs = "http://www.w3.org/2000/01/rdf-schema#"
val owl = "http://www.w3.org/2002/07/owl#"
val skos = "http://www.w3.org/2004/02/skos/core#"
val rico = "https://www.ica.org/standards/RiC/ontology#"
val ebucore = "http://www.ebu.ch/metadata/ontologies/ebucore/ebucore#"
val memint = "https://memobase.ch/institution/"
val memrs = "https://memobase.ch/recordSet/"
val memr = "https://memobase.ch/record/"
val mempo = "https://memobase.ch/instantiation/physical/"
val memdo = "https://memobase.ch/instantiation/digital/"
val dce = "http://purl.org/dc/elements/1.1/"
val dct = "http://purl.org/dc/terms/"
val schema = "http://schema.org/"
val foaf = "http://xmlns.com/foaf/0.1/"
val wdt = "http://www.wikidata.org/prop/direct/"
val wdtn = "http://www.wikidata.org/prop/direct-normalized/"
val rdau = "http://rdaregistry.info/Elements/u/"
}
\ No newline at end of file
/*
* mapper-service
* Copyright (C) 2020 Memoriav
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.memobase.rdf
import org.apache.jena.rdf.model.Property
import org.apache.jena.rdf.model.ResourceFactory
object RDA {
val hasSponsoringAgentOfResource = prop("P60451")
val hasFindingAid = prop("P60262")
val hasPlaceOfCapture = prop("P60556")
val hasColourContent = prop("P60558")
val hasProducer = prop("P60441")
private fun prop(name: String): Property {
return ResourceFactory.createProperty(NS.rdau, name)
}
}
\ No newline at end of file
/*
* mapper-service
* Copyright (C) 2020 Memoriav
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.memobase.rdf
import org.apache.jena.rdf.model.Property
import org.apache.jena.rdf.model.Resource
import org.apache.jena.rdf.model.ResourceFactory
object RDF {
val type: Property = prop("type")
private fun prop(name: String): Property {
return ResourceFactory.createProperty(NS.rdf, name)
}
private fun res(name: String): Resource {
return ResourceFactory.createResource(NS.rdf + name)
}
}
\ No newline at end of file
/*
* mapper-service
* Copyright (C) 2020 Memoriav
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.memobase.rdf
import org.apache.jena.rdf.model.Property
import org.apache.jena.rdf.model.Resource
import org.apache.jena.rdf.model.ResourceFactory
object RICO {
// Classes
val Record = res("Record")
val Instantiation = res("Instantiation")
val Title = res("Title")
val Language = res("Language")
val Identifier = res("Identifier")
val Place = res("Place")
val Agent = res("Agent")
val Person = res("Person")
val CorporateBody = res("CorporateBody")
val Rule = res("Rule")
val CarrierType = res("CarrierType")
val CreationRelation = res("CreationRelation")
val SingleDate = res("SingleDate")
val DateRange = res("DateRange")
val DateSet = res("DateSet")
// datatype properties
val title: Property = prop("title")
val source: Property = prop("source")
val descriptiveNote: Property = prop( "descriptiveNote")
val name: Property = prop( "name")
val type: Property = prop( "type")
val expressedDate: Property = prop( "expressedDate")
val normalizedDateValue: Property = prop( "normalizedDateValue")
val normalizedValue: Property = prop( "normalizedValue")
val scopeAndContent: Property = prop( "scopeAndContent")
val physicalCharacteristics: Property = prop( "physicalCharacteristics")
val conditionsOfUse: Property = prop("conditionsOfUse")
val conditionsOfAccess: Property = prop("conditionsOfAccess")
// Object Properties
val hasTitle: Property = prop( "hasTitle")
val isTitleOf: Property = prop( "isTitleOf")
val hasLanguage: Property = prop( "hasLanguage")
val isLanguageOf: Property = prop( "isLanguageOf")
val identifiedBy: Property = prop("identifiedBy")
val identifies: Property = prop("identifies")
val identifier: Property = prop("identifier")
val hasSubject: Property = prop( "hasSubject")
val hasCarrierType: Property = prop("hasCarrierType")
val publishedBy: Property = prop("publishedBy")
val heldBy: Property = prop( "heldBy")
val isPartOf: Property = prop( "isPartOf")
val hasProvenance: Property = prop("hasProvenance")
val instantiates: Property = prop("instantiates")
val hasInstantiation: Property = prop("hasInstantiation")
val hasDerivedInstantiation: Property = prop("hasDerivedInstantiation")
val isDerivedFromInstantiation: Property = prop("isDerivedFromInstantiation")
val creationRelationHasSource: Property = prop("creationRelationHasSource")
val creationRelationHasTarget: Property = prop("creationRelationHasTarget")
val agentIsTargetOfCreationRelation: Property = prop("agentIsTargetOfCreationRelation")
val recordResourceOrInstantiationIsSourceOfCreationRelation: Property = prop("recordResourceOrInstantiationIsSourceOfCreationRelation")
val regulates: Property = prop("regulates")
val intellectualPropertyRightsHeldBy: Property = prop("intellectualPropertyRightsHeldBy")
val regulatedBy: Property = prop("regulatedBy")
private fun prop(name: String): Property {
return ResourceFactory.createProperty(NS.rico, name)
}
private fun res(name: String): Resource {
return ResourceFactory.createResource(NS.rico + name)
}
}
\ No newline at end of file
/*
* mapper-service
* Copyright (C) 2020 Memoriav
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.memobase.rdf
import org.apache.jena.rdf.model.Property
import org.apache.jena.rdf.model.Resource
import org.apache.jena.rdf.model.ResourceFactory
import org.memobase.mapping.exceptions.InvalidMappingException
object SKOS {
val labelProperties = listOf("prefLabel")
val acceptedPropertiesList = listOf("prefLabel", "altLabel", "hiddenLabel",
"changeNote", "definition", "editorialNote", "example", "historyNote", "notation", "note", "scopeNote")
fun get(name: String): Property {
if (acceptedPropertiesList.contains(name)) {
return prop(name)
} else {
throw InvalidMappingException(
"Could not load skos property '$name' as it " +
"is not on the list of accepted properties: $acceptedPropertiesList."
)
}
}
// Classes
val ConceptScheme: Resource = res("ConceptScheme")
val Collection: Resource = res("Collection")
val OrderedCollection: Resource = res("OrderedCollection")
val Concept: Resource = res("Concept")
// Labels
val prefLabel: Property = prop("prefLabel")
val altLabel: Property = prop("altLabel")
val hiddenLabel: Property = prop("hiddenLabel")
// Annotation Properties
val changeNote: Property = prop("changeNote")
val definition: Property = prop("definition")
val editorialNote: Property = prop("editorialNote")
val example: Property = prop("example")
val historyNote: Property = prop("historyNote")
val notation: Property = prop("notation")
val note: Property = prop("note")
val scopeNote: Property = prop("scopeNote")
// Mapping Properties
val mappingRelation: Property = prop("mappingRelation")
val exactMatch: Property = prop("exactMatch")
val closeMatch: Property = prop("closeMatch")
val narrowMatch: Property = prop("narrowMatch")
val broadMatch: Property = prop("broadMatch")
// Semantic Relation Properties
val semanticRelation: Property = prop("semanticRelation")
val related: Property = prop("related")
val relatedMatch: Property = prop("relatedMatch")
// Hierarchy
val topConceptOf: Property = prop("topConceptOf")
val hasTopConcept: Property = prop("hasTopConcept")
val inScheme: Property = prop("inScheme")
val broader: Property = prop("broader")
val broaderTransitive: Property = prop("broaderTransitive")
val narrower: Property = prop("narrower")
val narrowerTransitive: Property = prop("narrowerTransitive")
// Collection Property
val member: Property = prop("member")
val memberList: Property = prop("memberList")
private fun prop(name: String): Property {
return ResourceFactory.createProperty(NS.skos, name)
}
private fun res(name: String): Resource {
return ResourceFactory.createResource(NS.skos + name)
}
}
\ No newline at end of file
<
/*
* record-parser
* Copyright (C) 2019 Memobase
*
* This program is free software: you can redistribute it and/or modify
......@@ -31,13 +30,14 @@ import org.junit.jupiter.params.provider.MethodSource
import org.memobase.helpers.StringHelpers
import org.memobase.mapping.MappingConfig
import org.memobase.mapping.fields.ConstantField
import org.memobase.params.IntegrationTestParams
import org.memobase.settings.SettingsLoader
import java.io.File
import java.nio.charset.Charset
import java.util.stream.Stream
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class Tests {
class IntegrationTests {
private val log = LogManager.getLogger("TestLogger")
private val resourcePath = "src/test/resources"
......@@ -47,22 +47,13 @@ class Tests {
private fun reportingTopic(value: String) = "$value-reporting"
@Test
fun `test mapping config validation`() {
val config = MappingConfig(configTestBasePath + "minimalValid")
assertThat(config).isNotNull
.hasFieldOrPropertyWithValue("uriField", "TestField")
.hasFieldOrPropertyWithValue("recordType", ConstantField("type", "Foto"))