Commit 9651516d authored by Jonas Waeber's avatar Jonas Waeber
Browse files

Add is published flag config to mapper service

parent 917b2f28
Pipeline #13998 passed with stages
in 7 minutes and 28 seconds
......@@ -32,7 +32,7 @@ ext {
}
dependencies {
implementation 'org.memobase:memobase-service-utilities:1.12.1'
implementation 'org.memobase:memobase-service-utilities:0.14.2'
implementation 'ch.memobase:import-process-effects-registry_2.12:0.2.1'
// Logging Framework
implementation "org.apache.logging.log4j:log4j-api:${log4jV}"
......
......@@ -7,6 +7,7 @@ data:
APPLICATION_ID: "{{ .Values.processId }}-{{ .Values.jobName }}"
INSTITUTION_ID: "{{ .Values.institutionId }}"
RECORD_SET_ID: "{{ .Values.recordSetId }}"
IS_PUBLISHED: "{{ .Values.isPublished }}"
TOPIC_IN: "{{ .Values.processId }}-{{ .Values.lastJobName }}"
TOPIC_OUT: "{{ .Values.processId }}-{{ .Values.jobName }}"
TOPIC_PROCESS: "{{ .Values.processId }}-reporting"
\ No newline at end of file
......@@ -21,6 +21,7 @@ processId: p0001
institutionId: placeholder
recordSetId: placeholder
isPublished: false
#the filename of the mapping
configFileName: mapping.yml
......@@ -37,6 +37,9 @@ class KafkaTopology(
private val log = LogManager.getLogger("KafkaTopology")
private val config = MappingConfig(settings.appSettings.getProperty("configs"))
private val reportTopic = settings.outputTopic + "-reporting"
private val institutionId = settings.appSettings.getProperty("institutionId")
private val recordSetId = settings.appSettings.getProperty("recordSetId")
private val isPublished = settings.appSettings.getProperty("isPublished")!!.toBoolean()
fun prepare(): StreamsBuilder {
val builder = StreamsBuilder()
......@@ -101,9 +104,10 @@ class KafkaTopology(
config.physicalObjectFieldMappers,
config.digitalObjectFieldMappers,
value,
settings.appSettings.getProperty("institutionId"),
settings.appSettings.getProperty("recordSetId"),
config.sponsoredByMemoriav
institutionId,
recordSetId,
config.sponsoredByMemoriav,
isPublished
)
}
......
......@@ -35,7 +35,8 @@ class Service(file: String = "app.yml") {
listOf(
"institutionId",
"recordSetId",
"configs"
"configs",
"isPublished"
),
file,
useStreamsConfig = true
......
......@@ -18,18 +18,28 @@
package org.memobase.builder
import org.apache.jena.datatypes.RDFDatatype
import org.apache.jena.rdf.model.Resource
import org.memobase.helpers.StringHelpers
import org.memobase.mapping.KEYS
import org.memobase.rdf.NS
import org.memobase.rdf.RDA
import org.memobase.rdf.RICO
import rdf.MB
class Record(sourceId: String, type: String, recordSetId: String, institutionId: String, hasSponsoringAgent: Boolean) :
class Record(
sourceId: String,
type: String,
recordSetId: String,
institutionId: String,
hasSponsoringAgent: Boolean,
isPublished: Boolean
) :
RecordResource(institutionId) {
private val id = recordSetId + "-" + StringHelpers.normalizeId(sourceId)
override val resource: Resource = model.createResource(NS.mbr + id)
init {
addRdfType(RICO.Record)
resource.addProperty(RICO.type, type)
......@@ -39,6 +49,7 @@ class Record(sourceId: String, type: String, recordSetId: String, institutionId:
if (hasSponsoringAgent) {
resource.addProperty(RDA.hasSponsoringAgentOfResource, model.createResource(KEYS.memoriavUri))
}
resource.addProperty(MB.isPublished, model.createTypedLiteral(isPublished))
}
fun addInstantiation(instantiation: Instantiation) {
......
......@@ -39,7 +39,8 @@ class ResourceBuilder(
private val source: Map<String, String>,
private val institutionId: String,
private val recordSetId: String,
private val isSponsoredByMemoriav: Boolean
private val isSponsoredByMemoriav: Boolean,
private val isPublished: Boolean
) {
private var record: Record? = null
private var physicalObject: PhysicalObject? = null
......@@ -91,7 +92,7 @@ class ResourceBuilder(
}
fun generateRecord(): ResourceBuilder {
record = Record(recordId, recordTypeValue, recordSetId, institutionId, isSponsoredByMemoriav)
record = Record(recordId, recordTypeValue, recordSetId, institutionId, isSponsoredByMemoriav, isPublished)
for (recordFieldMapper in recordFieldMappers) {
recordFieldMapper.apply(source, record!!)
}
......
app:
institutionId: ${INSTITUTION_ID:?system}
recordSetId: ${RECORD_SET_ID:?system}
institutionId: ${INSTITUTION_ID:?user}
recordSetId: ${RECORD_SET_ID:?user}
configs: /configs/mappings
isPublished: ${IS_PUBLISHED:?user}
kafka:
streams:
bootstrap.servers: ${KAFKA_BOOTSTRAP_SERVERS:?system}
......
......@@ -48,15 +48,8 @@ class IntegrationTests {
@ParameterizedTest
@MethodSource("kafkaTests")
fun `test kafka topology`(params: IntegrationTestParams) {
val settings = SettingsLoader(
listOf(
"institutionId",
"recordSetId",
"configs"
),
fileName = "kafkaTest${params.count}.yml",
useStreamsConfig = true
)
val service = Service("kafkaTest${params.count}.yml")
val settings = service.settings
val testDriver = TopologyTestDriver(KafkaTopology(settings).prepare().build(), settings.kafkaStreamsSettings)
val factory = ConsumerRecordFactory(
StringSerializer(), StringSerializer()
......
......@@ -36,19 +36,30 @@ class TestMapperFunctions {
@Test
fun `test agent mapper`() {
val agentMapper = AgentFieldMapper()
agentMapper.setFields("creator",
listOf(DirectMapField("name", "person.name"),
ListField("relationName", listOf(
agentMapper.setFields(
"creator",
listOf(
DirectMapField("name", "person.name"),
ListField(
"relationName", listOf(
ConstantField("relationName", "Schauspieler"),
ConstantField("relationName", "DUMMY-VALUE")
))
)
)
)
)
agentMapper.sourceKey = "creators"
val source: Map<String, Any> = mapOf(
Pair("person", Pair("name", "TEST NAME"))
)
val record = Record("1", "Foto", "rs1", "ins1", true)
val record = Record(
"1",
"Foto",
"rs1",
"ins1",
hasSponsoringAgent = true,
isPublished = false
)
agentMapper.apply(source, record)
......
......@@ -2,6 +2,7 @@ app:
institutionId: "BAZ"
recordSetId: "BAZ-B_MEI"
configs: src/test/resources/kafkaTests/1/config
isPublished: false
kafka:
streams:
bootstrap.servers: localhost:12345
......
......@@ -2,6 +2,7 @@ app:
institutionId: "TEST"
recordSetId: "TEST_RECORD_SET"
configs: src/test/resources/kafkaTests/2/config
isPublished: false
kafka:
streams:
bootstrap.servers: localhost:12345
......
......@@ -2,6 +2,7 @@ app:
institutionId: "TEST"
recordSetId: "TEST_RECORD_SET"
configs: src/test/resources/kafkaTests/3/config
isPublished: false
kafka:
streams:
bootstrap.servers: localhost:12345
......
......@@ -2,6 +2,7 @@ app:
institutionId: "TEST"
recordSetId: "TEST_RECORD_SET"
configs: src/test/resources/kafkaTests/4/config
isPublished: false
kafka:
streams:
bootstrap.servers: localhost:12345
......
......@@ -2,6 +2,7 @@ app:
institutionId: "TEST"
recordSetId: "TEST_RECORD_SET"
configs: src/test/resources/kafkaTests/5/config
isPublished: false
kafka:
streams:
bootstrap.servers: localhost:12345
......
......@@ -2,6 +2,7 @@ app:
institutionId: "TEST"
recordSetId: "TEST_RECORD_SET"
configs: src/test/resources/kafkaTests/6/config
isPublished: false
kafka:
streams:
bootstrap.servers: localhost:12345
......
......@@ -2,6 +2,7 @@ app:
institutionId: "TEST"
recordSetId: "TEST_RECORD_SET"
configs: src/test/resources/kafkaTests/7/config
isPublished: false
kafka:
streams:
bootstrap.servers: localhost:12345
......
......@@ -2,6 +2,7 @@ app:
institutionId: "TEST"
recordSetId: "TEST_RECORD_SET"
configs: src/test/resources/kafkaTests/8/config
isPublished: false
kafka:
streams:
bootstrap.servers: localhost:12345
......
......@@ -2,6 +2,7 @@ app:
institutionId: "TEST"
recordSetId: "TEST_RECORD_SET"
configs: src/test/resources/kafkaTests/9/config
isPublished: false
kafka:
streams:
bootstrap.servers: localhost:12345
......
......@@ -3,6 +3,7 @@
<https://memobase.ch/physical/BAZ-B_MEI-MEI_49884-1> <https://www.ica.org/standards/RiC/ontology#identifiedBy> _:B .
<https://memobase.ch/physical/BAZ-B_MEI-MEI_49884-1> <https://www.ica.org/standards/RiC/ontology#instantiates> <https://memobase.ch/record/BAZ-B_MEI-MEI_49884> .
<https://memobase.ch/physical/BAZ-B_MEI-MEI_49884-1> <https://www.ica.org/standards/RiC/ontology#type> "physicalObject" .
<https://memobase.ch/record/BAZ-B_MEI-MEI_49884> <http://memobase.ch/internal/isPublished> "false"^^<http://www.w3.org/2001/XMLSchema#boolean> .
<https://memobase.ch/record/BAZ-B_MEI-MEI_49884> <http://rdaregistry.info/Elements/u/P60451> <https://memobase.ch/institution/mrv> .
<https://memobase.ch/record/BAZ-B_MEI-MEI_49884> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Record> .
<https://memobase.ch/record/BAZ-B_MEI-MEI_49884> <https://www.ica.org/standards/RiC/ontology#hasInstantiation> <https://memobase.ch/physical/BAZ-B_MEI-MEI_49884-1> .
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment