Commit 3fb487d2 authored by Jonas Waeber's avatar Jonas Waeber

Improve logging

parent ab480e9a
Pipeline #21799 passed with stage
in 2 minutes and 15 seconds
......@@ -78,7 +78,7 @@ sealed class ConfigField {
for (field in fields) {
when (field) {
is MappedAnnotationField ->
FieldParsers.unpackSource(field.field, source).let { value ->
FieldParsers.unpackSource(field.key, field.field, source).let { value ->
when (value) {
is SimpleString ->
if (resultList.size == 1) {
......@@ -126,7 +126,7 @@ sealed class ConfigField {
languagePair.sources.forEach { field ->
when (field) {
is MappedAnnotationField ->
FieldParsers.unpackSource(field.field, source).let { value ->
FieldParsers.unpackSource(field.key, field.field, source).let { value ->
when (value) {
is SimpleString ->
if (resultList.size == 1) {
......
......@@ -19,6 +19,7 @@
package ch.memobase.mapping.fields
import ch.memobase.exceptions.InvalidMappingException
import ch.memobase.mapping.KEYS
import ch.memobase.mapping.fields.SourceElement.Empty
import ch.memobase.mapping.fields.SourceElement.SimpleString
import ch.memobase.mapping.fields.SourceElement.StringList
......@@ -154,29 +155,28 @@ object FieldParsers {
* Currently only Strings, Lists of Strings and Objects can be parsed.
* Objects may only contain Strings or List of Strings.
*/
fun unpackSource(field: String, source: Map<String, Any>): SourceElement {
fun unpackSource(key: String, field: String, source: Map<String, Any>): SourceElement {
return if (field.contains('.')) {
val fields = field.split('.')
source[fields[0]].let { objectValue ->
when (objectValue) {
is Map<*, *> -> {
if (objectValue.containsKey(fields[1])) {
unpackValue(field, objectValue[fields[1]])
unpackValue(key, field, objectValue[fields[1]])
} else {
Empty
}
}
is List<*> -> {
// TODO:
when {
objectValue.isEmpty() -> Empty
objectValue.size == 1 -> {
val item = objectValue[0]
if (item is Map<*, *>) {
unpackValue(field, item[fields[1]])
unpackValue(key, field, item[fields[1]])
} else {
log.error(
"The value in field ${field[0]} inside of the array is " +
"The value in field ${field[0]} from key $key inside of the array is " +
"not an object as expected."
)
Empty
......@@ -186,14 +186,14 @@ object FieldParsers {
val resultList = mutableListOf<String>()
for (item in objectValue) {
if (item is Map<*, *>) {
when (val unpackedValue = unpackValue(field, item[fields[1]])) {
when (val unpackedValue = unpackValue(key, field, item[fields[1]])) {
is SimpleString -> resultList.add(unpackedValue.value)
is StringList -> resultList.addAll(unpackedValue.value)
Empty -> {
// Adding an empty value here because optional fields in objects
// otherwise mess up the order. This means that empty fields need to be ignored.
resultList.add("")
log.debug("No or illegal value in map found.")
log.debug("No or illegal value in map found for $key with target field $field.")
}
}
}
......@@ -204,30 +204,30 @@ object FieldParsers {
}
null -> Empty
else -> {
log.error("Could not parse object for field $field in source.")
log.error("Could not parse object for field $field in source from key $key.")
Empty
}
}
}
} else {
unpackValue(field, source[field])
unpackValue(key, field, source[field])
}
}
private fun unpackValue(field: String, value: Any?): SourceElement {
private fun unpackValue(key: String, field: String, value: Any?): SourceElement {
return when (value) {
is String -> SimpleString(value)
is List<*> -> {
if (value.isNotEmpty() && value[0] is String) {
StringList(value as List<String>)
} else {
log.error("Could not parse list elements in field $field. It is either an empty list or is not a string.")
log.error("Could not parse list elements in field $field from key $key. It is either an empty list or is not a string.")
Empty
}
}
null -> Empty
else -> {
log.error("Could not parse element in field $field. The value is neither a string nor null.")
log.error("Could not parse element in field $field from key $key. The value is neither a string nor null.")
Empty
}
}
......
......@@ -36,7 +36,7 @@ class CarrierTypeMapper(val field: AnnotationField) : AbstractFieldMapper() {
override fun apply(source: Map<String, Any>, subject: IResource) {
when (field) {
is MappedAnnotationField ->
FieldParsers.unpackSource(field.field, source).let { sourceElement: SourceElement ->
FieldParsers.unpackSource(field.key, field.field, source).let { sourceElement: SourceElement ->
when (sourceElement) {
is SimpleString -> subject.addRicoCarrierType(listOf(field.toLiteral(sourceElement.value)))
is StringList ->
......
......@@ -30,7 +30,7 @@ import org.apache.logging.log4j.LogManager
class DateFieldMapper(private val directMapField: DirectMapField) : AbstractFieldMapper() {
private val log = LogManager.getLogger(this::class.java)
override fun apply(source: Map<String, Any>, subject: IResource) {
FieldParsers.unpackSource(directMapField.field, source).let { sourceElement: SourceElement ->
FieldParsers.unpackSource(directMapField.key, directMapField.field, source).let { sourceElement: SourceElement ->
when (sourceElement) {
is SimpleString -> subject.addDate(directMapField.key, sourceElement.value)
is StringList -> sourceElement.value.forEach {
......
......@@ -30,7 +30,7 @@ import org.apache.logging.log4j.LogManager
class DirectFieldMapper(private val directMapField: DirectMapField) : AbstractFieldMapper() {
private val log = LogManager.getLogger(this::class.java)
override fun apply(source: Map<String, Any>, subject: IResource) {
FieldParsers.unpackSource(directMapField.field, source).let { sourceElement: SourceElement ->
FieldParsers.unpackSource(directMapField.key, directMapField.field, source).let { sourceElement: SourceElement ->
when (sourceElement) {
is SimpleString -> subject.addLiteral(directMapField.key, directMapField.toLiteral(sourceElement.value))
is StringList -> sourceElement.value.forEach {
......
......@@ -43,7 +43,7 @@ class ExpandedRuleFieldMapper(
configField.forEach { field ->
when (field) {
is MappedAnnotationField ->
FieldParsers.unpackSource(field.field, source).let { sourceElement: SourceElement ->
FieldParsers.unpackSource(field.key, field.field, source).let { sourceElement: SourceElement ->
when (sourceElement) {
is SimpleString ->
if (properties.size == 1)
......
......@@ -30,7 +30,7 @@ import org.apache.logging.log4j.LogManager
class PrefixFieldMapper(private val prefixField: PrefixField) : AbstractFieldMapper() {
private val log = LogManager.getLogger(this::class.java)
override fun apply(source: Map<String, Any>, subject: IResource) {
FieldParsers.unpackSource(prefixField.field, source).let { sourceElement: SourceElement ->
FieldParsers.unpackSource(prefixField.key, prefixField.field, source).let { sourceElement: SourceElement ->
when (sourceElement) {
is SimpleString -> subject.addLiteral(prefixField.key, prefixField.toLiteral(sourceElement.value))
is StringList -> sourceElement.value.forEach {
......
......@@ -36,7 +36,7 @@ class RicoConceptMapper(private val rdfType: String, val field: AnnotationField)
override fun apply(source: Map<String, Any>, subject: IResource) {
when (field) {
is MappedAnnotationField ->
FieldParsers.unpackSource(field.field, source).let {
FieldParsers.unpackSource(field.key, field.field, source).let {
when (it) {
is SimpleString -> subject.addRicoConcept(rdfType, field.key, listOf(field.toLiteral(it.value)))
is StringList ->
......
......@@ -36,7 +36,7 @@ class RuleFieldMapper(private val configField: ConfigField) : AbstractFieldMappe
override fun apply(source: Map<String, Any>, subject: IResource) {
when (configField) {
is MappedAnnotationField ->
FieldParsers.unpackSource(configField.field, source).let {
FieldParsers.unpackSource(configField.key, configField.field, source).let {
when (it) {
is SimpleString -> subject.addRule(
configField.key,
......
......@@ -44,7 +44,7 @@ abstract class TypeFieldMapper : AbstractFieldMapper() {
}
private fun addMappedField(field: MappedAnnotationField, source: Map<String, Any>) {
FieldParsers.unpackSource(field.field, source).let { sourceElement ->
FieldParsers.unpackSource(field.key, field.field, source).let { sourceElement ->
when (sourceElement) {
is SimpleString ->
if (properties.size == 1) {
......@@ -55,7 +55,7 @@ abstract class TypeFieldMapper : AbstractFieldMapper() {
is StringList ->
sourceElement.value.forEachIndexed { index, s ->
if (s.isNotEmpty()) {
if (properties.size <= index + 1) {
if (properties.size >= index + 1) {
properties[index].add(Pair(field.key, field.toLiteral(s)))
} else {
properties.add(index, mutableListOf(Pair(field.key, field.toLiteral(s))))
......
......@@ -19,12 +19,18 @@
package ch.memobase.test
import ch.memobase.builder.Record
import ch.memobase.builder.ResourceBuilder
import ch.memobase.mapping.MapperParsers
import ch.memobase.mapping.MappingConfigurationParser
import ch.memobase.rdf.RICO
import ch.memobase.settings.HeaderMetadata
import com.beust.klaxon.Klaxon
import java.io.File
import java.io.FileOutputStream
import org.apache.jena.rdf.model.impl.SelectorImpl
import org.apache.jena.riot.Lang
import org.apache.jena.riot.RDFDataMgr
import org.apache.jena.riot.RDFFormat
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.TestInstance
......@@ -33,6 +39,18 @@ import org.junit.jupiter.api.assertAll
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class TestAgentMapper {
private val klaxon = Klaxon()
private val headerMetadata = HeaderMetadata(
"test-001",
"1",
"test",
false,
"record",
"identifierMain",
0, 0, 0, 0
)
private val mapper = MapperParsers.buildAgentMapper(
"creators",
listOf(
......@@ -47,7 +65,35 @@ class TestAgentMapper {
)
)[0]
private val regex = Regex("(_:B[A-Za-z0-9]+)")
private val regexTime = Regex("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}")
private fun sort(source: List<String>): String {
return source.map {
var replacedString = it
for (matchResult in regex.findAll(it)) {
replacedString = replacedString.replace(matchResult.groups[0]?.value.orEmpty(), "_:B")
}
for (matchResult in regexTime.findAll(it)) {
replacedString = replacedString.replace(matchResult.groups[0]?.value.orEmpty(), "2020-10-10T09:10:22")
}
replacedString
}.sorted().reduce { acc, s -> acc + "\n" + s }.trim()
}
private val resourcePath = "src/test/resources/turtleOutput"
private val inputPath = "src/test/resources/agent-mapper"
private fun readInputFile(fileName: String): String {
return File("$inputPath/$fileName.json").readText()
}
private fun readOutputFile(fileName: String): String {
return sort(File("$inputPath/$fileName.nt").readLines())
}
private fun readMapping(fileName: String): ByteArray {
return File("$inputPath/$fileName.yml").readBytes()
}
// TODO: Actually test this.
@Test
......@@ -155,4 +201,36 @@ class TestAgentMapper {
Lang.TURTLE
)
}
@Test
fun `test multiple contributor names`() {
val source = klaxon.parse<Map<String, Any>>(readInputFile("input-multiple-contributors")).orEmpty()
val mapping = MappingConfigurationParser(readMapping("mapping-multiple-contributors"))
val configuration = mapping.get()
val result = ResourceBuilder(
source,
configuration,
headerMetadata.institutionId,
headerMetadata.recordSetId,
headerMetadata.isPublished
).extractRecordId()
.extractRecordTypeValue()
.generateRecord()
.generatePhysicalObject()
.generateDigitalObject()
.addDerivedConnection()
val ntriples = result
.writeRecord(RDFFormat.NTRIPLES_UTF8)
val turtle = result.writeRecord(RDFFormat.TURTLE_PRETTY)
assertThat(sort(ntriples.second.split("\n")))
.isEqualTo(readOutputFile("output-multiple-contributors"))
FileOutputStream(File("$resourcePath/output-multiple-contributors.ttl")).use {
it.bufferedWriter().use { writer ->
writer.write(turtle.second)
}
}
}
}
{
"id": "identifier",
"contributorPerson": [
{
"name": "Simon Epiney / Conseiller national PDC Valais",
"role": "Gesprächspartner"
},
{
"name": "Markus Ruf / Conseiller national Démocrates suisses Berne",
"role": "Gesprächspartner"
},
{
"name": "Jean Cavadini / Conseiller aux Etats PL Neuchâtel",
"role": "Gesprächspartner"
},
{
"name": "Irène Gardiol / Conseillère nationale écologiste Vaud",
"role": "Gesprächspartner"
},
{
"name": "Flavio Maspoli / Conseiller national Lega Tessin",
"role": "Gesprächspartner"
},
{
"name": "Flavio Cotti / Conseiller fédéral DFAE",
"role": "Gesprächspartner"
},
{
"name": "Suzette Sandoz / Conseillère nationale PL Vaud",
"role": "Gesprächspartner"
},
{
"name": "Matthias Krafft / Directeur Direction droit international public DFAE",
"role": "Gesprächspartner"
},
{
"name": "Vital Darbellay / Conseiller national PDC Valais",
"role": "Gesprächspartner"
},
{
"name": "Jacques Martin / Conseiller aux Etats PRD Vaud",
"role": "Gesprächspartner"
},
{
"name": "Pierre-André Tschanz / Journaliste RSI"
},
{
"name": "Pierre-André Tschanz / Journaliste RSI",
"role": "Interviewer"
},
{
"name": "Catherine Miskiewicz / Journaliste RSI"
},
{
"name": "Catherine Miskiewicz",
"role": "ModeratorIn"
},
{
"name": "Pierre-André Tschanz",
"role": "ModeratorIn"
},
{
"name": "Pierre-André Tschanz",
"role": "Autor"
},
{
"name": "Catherine Miskiewicz",
"role": "Autor"
}
]
}
\ No newline at end of file
record:
uri: id
type:
const: Foto
contributors:
person:
name: contributorPerson.name
relationName: contributorPerson.role
\ No newline at end of file
......@@ -5,15 +5,6 @@
@prefix ebucore: <http://www.ebu.ch/metadata/ontologies/ebucore/ebucore#> .
@prefix skos: <http://www.w3.org/2004/02/skos/core#> .
_:b0 a rico:CreationRelation ;
rico:creationRelationHasSource <https://memobase.ch/record/rs1-1> ;
rico:creationRelationHasTarget [ a rico:Person ;
rico:agentIsTargetOfCreationRelation
_:b0 ;
rico:name "Hans Zimmer"
] ;
rico:type "creator" .
<https://memobase.ch/record/rs1-1>
a rico:Record ;
rdau:P60451 <https://memobase.ch/institution/mrv> ;
......@@ -28,3 +19,12 @@ _:b0 a rico:CreationRelation ;
rico:recordResourceOrInstantiationIsSourceOfCreationRelation
_:b0 ;
rico:type "Foto" .
_:b0 a rico:CreationRelation ;
rico:creationRelationHasSource <https://memobase.ch/record/rs1-1> ;
rico:creationRelationHasTarget [ a rico:Person ;
rico:agentIsTargetOfCreationRelation
_:b0 ;
rico:name "Hans Zimmer"
] ;
rico:type "creator" .
......@@ -5,16 +5,6 @@
@prefix ebucore: <http://www.ebu.ch/metadata/ontologies/ebucore/ebucore#> .
@prefix skos: <http://www.w3.org/2004/02/skos/core#> .
_:b0 a rico:CreationRelation ;
rico:creationRelationHasSource <https://memobase.ch/record/rs1-1> ;
rico:creationRelationHasTarget [ a rico:Person ;
rico:agentIsTargetOfCreationRelation
_:b0 ;
rico:name "First Person"
] ;
rico:name "Relation 1" ;
rico:type "creator" .
<https://memobase.ch/record/rs1-1>
a rico:Record ;
rdau:P60451 <https://memobase.ch/institution/mrv> ;
......@@ -27,24 +17,34 @@ _:b0 a rico:CreationRelation ;
] ;
rico:isPartOf <https://memobase.ch/recordSet/rs1> ;
rico:recordResourceOrInstantiationIsSourceOfCreationRelation
_:b1 , _:b0 , _:b2 ;
_:b0 , _:b1 , _:b2 ;
rico:type "Foto" .
_:b2 a rico:CreationRelation ;
_:b1 a rico:CreationRelation ;
rico:creationRelationHasSource <https://memobase.ch/record/rs1-1> ;
rico:creationRelationHasTarget [ a rico:Person ;
rico:agentIsTargetOfCreationRelation
_:b2 ;
rico:name "Second Person"
_:b1 ;
rico:name "Third Person"
] ;
rico:name "Relation 3" ;
rico:type "creator" .
_:b0 a rico:CreationRelation ;
rico:creationRelationHasSource <https://memobase.ch/record/rs1-1> ;
rico:creationRelationHasTarget [ a rico:Person ;
rico:agentIsTargetOfCreationRelation
_:b0 ;
rico:name "First Person"
] ;
rico:name "Relation 1" ;
rico:type "creator" .
_:b1 a rico:CreationRelation ;
_:b2 a rico:CreationRelation ;
rico:creationRelationHasSource <https://memobase.ch/record/rs1-1> ;
rico:creationRelationHasTarget [ a rico:Person ;
rico:agentIsTargetOfCreationRelation
_:b1 ;
rico:name "Third Person"
_:b2 ;
rico:name "Second Person"
] ;
rico:name "Relation 3" ;
rico:type "creator" .
@prefix rdau: <http://rdaregistry.info/Elements/u/> .
@prefix dct: <http://purl.org/dc/terms/> .
@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
@prefix rico: <https://www.ica.org/standards/RiC/ontology#> .
@prefix ebucore: <http://www.ebu.ch/metadata/ontologies/ebucore/ebucore#> .
@prefix skos: <http://www.w3.org/2004/02/skos/core#> .
_:b0 a rico:CreationRelation ;
rico:creationRelationHasSource <https://memobase.ch/record/test-001-identifier> ;
rico:creationRelationHasTarget [ a rico:Person ;
rico:agentIsTargetOfCreationRelation
_:b0 ;
rico:name "Flavio Maspoli / Conseiller national Lega Tessin"
] ;
rico:name "Gesprächspartner" ;
rico:type "contributor" .
_:b1 a rico:CreationRelation ;
rico:creationRelationHasSource <https://memobase.ch/record/test-001-identifier> ;
rico:creationRelationHasTarget [ a rico:Person ;
rico:agentIsTargetOfCreationRelation
_:b1 ;
rico:name "Suzette Sandoz / Conseillère nationale PL Vaud"
] ;
rico:name "Gesprächspartner" ;
rico:type "contributor" .
_:b2 a rico:CreationRelation ;
rico:creationRelationHasSource <https://memobase.ch/record/test-001-identifier> ;
rico:creationRelationHasTarget [ a rico:Person ;
rico:agentIsTargetOfCreationRelation
_:b2 ;
rico:name "Flavio Cotti / Conseiller fédéral DFAE"
] ;
rico:name "Gesprächspartner" ;
rico:type "contributor" .
_:b3 a rico:CreationRelation ;
rico:creationRelationHasSource <https://memobase.ch/record/test-001-identifier> ;
rico:creationRelationHasTarget [ a rico:Person ;
rico:agentIsTargetOfCreationRelation
_:b3 ;
rico:name "Jacques Martin / Conseiller aux Etats PRD Vaud"
] ;
rico:name "Gesprächspartner" ;
rico:type "contributor" .
_:b4 a rico:CreationRelation ;
rico:creationRelationHasSource <https://memobase.ch/record/test-001-identifier> ;
rico:creationRelationHasTarget [ a rico:Person ;
rico:agentIsTargetOfCreationRelation
_:b4 ;
rico:name "Markus Ruf / Conseiller national Démocrates suisses Berne"
] ;
rico:name "Gesprächspartner" ;
rico:type "contributor" .
_:b5 a rico:CreationRelation ;
rico:creationRelationHasSource <https://memobase.ch/record/test-001-identifier> ;
rico:creationRelationHasTarget [ a rico:Person ;
rico:agentIsTargetOfCreationRelation
_:b5 ;
rico:name "Catherine Miskiewicz / Journaliste RSI"
] ;
rico:type "contributor" .
_:b6 a rico:CreationRelation ;
rico:creationRelationHasSource <https://memobase.ch/record/test-001-identifier> ;
rico:creationRelationHasTarget [ a rico:Person ;
rico:agentIsTargetOfCreationRelation
_:b6 ;
rico:name "Irène Gardiol / Conseillère nationale écologiste Vaud"
] ;
rico:name "Gesprächspartner" ;
rico:type "contributor" .
_:b7 a rico:CreationRelation ;
rico:creationRelationHasSource <https://memobase.ch/record/test-001-identifier> ;
rico:creationRelationHasTarget [ a rico:Person ;
rico:agentIsTargetOfCreationRelation
_:b7 ;
rico:name "Simon Epiney / Conseiller national PDC Valais"