Commit 3fb487d2 authored by Jonas Waeber's avatar Jonas Waeber
Browse files

Improve logging

parent ab480e9a
Pipeline #21799 passed with stage
in 2 minutes and 15 seconds
...@@ -78,7 +78,7 @@ sealed class ConfigField { ...@@ -78,7 +78,7 @@ sealed class ConfigField {
for (field in fields) { for (field in fields) {
when (field) { when (field) {
is MappedAnnotationField -> is MappedAnnotationField ->
FieldParsers.unpackSource(field.field, source).let { value -> FieldParsers.unpackSource(field.key, field.field, source).let { value ->
when (value) { when (value) {
is SimpleString -> is SimpleString ->
if (resultList.size == 1) { if (resultList.size == 1) {
...@@ -126,7 +126,7 @@ sealed class ConfigField { ...@@ -126,7 +126,7 @@ sealed class ConfigField {
languagePair.sources.forEach { field -> languagePair.sources.forEach { field ->
when (field) { when (field) {
is MappedAnnotationField -> is MappedAnnotationField ->
FieldParsers.unpackSource(field.field, source).let { value -> FieldParsers.unpackSource(field.key, field.field, source).let { value ->
when (value) { when (value) {
is SimpleString -> is SimpleString ->
if (resultList.size == 1) { if (resultList.size == 1) {
......
...@@ -19,6 +19,7 @@ ...@@ -19,6 +19,7 @@
package ch.memobase.mapping.fields package ch.memobase.mapping.fields
import ch.memobase.exceptions.InvalidMappingException import ch.memobase.exceptions.InvalidMappingException
import ch.memobase.mapping.KEYS
import ch.memobase.mapping.fields.SourceElement.Empty import ch.memobase.mapping.fields.SourceElement.Empty
import ch.memobase.mapping.fields.SourceElement.SimpleString import ch.memobase.mapping.fields.SourceElement.SimpleString
import ch.memobase.mapping.fields.SourceElement.StringList import ch.memobase.mapping.fields.SourceElement.StringList
...@@ -154,29 +155,28 @@ object FieldParsers { ...@@ -154,29 +155,28 @@ object FieldParsers {
* Currently only Strings, Lists of Strings and Objects can be parsed. * Currently only Strings, Lists of Strings and Objects can be parsed.
* Objects may only contain Strings or List of Strings. * Objects may only contain Strings or List of Strings.
*/ */
fun unpackSource(field: String, source: Map<String, Any>): SourceElement { fun unpackSource(key: String, field: String, source: Map<String, Any>): SourceElement {
return if (field.contains('.')) { return if (field.contains('.')) {
val fields = field.split('.') val fields = field.split('.')
source[fields[0]].let { objectValue -> source[fields[0]].let { objectValue ->
when (objectValue) { when (objectValue) {
is Map<*, *> -> { is Map<*, *> -> {
if (objectValue.containsKey(fields[1])) { if (objectValue.containsKey(fields[1])) {
unpackValue(field, objectValue[fields[1]]) unpackValue(key, field, objectValue[fields[1]])
} else { } else {
Empty Empty
} }
} }
is List<*> -> { is List<*> -> {
// TODO:
when { when {
objectValue.isEmpty() -> Empty objectValue.isEmpty() -> Empty
objectValue.size == 1 -> { objectValue.size == 1 -> {
val item = objectValue[0] val item = objectValue[0]
if (item is Map<*, *>) { if (item is Map<*, *>) {
unpackValue(field, item[fields[1]]) unpackValue(key, field, item[fields[1]])
} else { } else {
log.error( log.error(
"The value in field ${field[0]} inside of the array is " + "The value in field ${field[0]} from key $key inside of the array is " +
"not an object as expected." "not an object as expected."
) )
Empty Empty
...@@ -186,14 +186,14 @@ object FieldParsers { ...@@ -186,14 +186,14 @@ object FieldParsers {
val resultList = mutableListOf<String>() val resultList = mutableListOf<String>()
for (item in objectValue) { for (item in objectValue) {
if (item is Map<*, *>) { if (item is Map<*, *>) {
when (val unpackedValue = unpackValue(field, item[fields[1]])) { when (val unpackedValue = unpackValue(key, field, item[fields[1]])) {
is SimpleString -> resultList.add(unpackedValue.value) is SimpleString -> resultList.add(unpackedValue.value)
is StringList -> resultList.addAll(unpackedValue.value) is StringList -> resultList.addAll(unpackedValue.value)
Empty -> { Empty -> {
// Adding an empty value here because optional fields in objects // Adding an empty value here because optional fields in objects
// otherwise mess up the order. This means that empty fields need to be ignored. // otherwise mess up the order. This means that empty fields need to be ignored.
resultList.add("") resultList.add("")
log.debug("No or illegal value in map found.") log.debug("No or illegal value in map found for $key with target field $field.")
} }
} }
} }
...@@ -204,30 +204,30 @@ object FieldParsers { ...@@ -204,30 +204,30 @@ object FieldParsers {
} }
null -> Empty null -> Empty
else -> { else -> {
log.error("Could not parse object for field $field in source.") log.error("Could not parse object for field $field in source from key $key.")
Empty Empty
} }
} }
} }
} else { } else {
unpackValue(field, source[field]) unpackValue(key, field, source[field])
} }
} }
private fun unpackValue(field: String, value: Any?): SourceElement { private fun unpackValue(key: String, field: String, value: Any?): SourceElement {
return when (value) { return when (value) {
is String -> SimpleString(value) is String -> SimpleString(value)
is List<*> -> { is List<*> -> {
if (value.isNotEmpty() && value[0] is String) { if (value.isNotEmpty() && value[0] is String) {
StringList(value as List<String>) StringList(value as List<String>)
} else { } else {
log.error("Could not parse list elements in field $field. It is either an empty list or is not a string.") log.error("Could not parse list elements in field $field from key $key. It is either an empty list or is not a string.")
Empty Empty
} }
} }
null -> Empty null -> Empty
else -> { else -> {
log.error("Could not parse element in field $field. The value is neither a string nor null.") log.error("Could not parse element in field $field from key $key. The value is neither a string nor null.")
Empty Empty
} }
} }
......
...@@ -36,7 +36,7 @@ class CarrierTypeMapper(val field: AnnotationField) : AbstractFieldMapper() { ...@@ -36,7 +36,7 @@ class CarrierTypeMapper(val field: AnnotationField) : AbstractFieldMapper() {
override fun apply(source: Map<String, Any>, subject: IResource) { override fun apply(source: Map<String, Any>, subject: IResource) {
when (field) { when (field) {
is MappedAnnotationField -> is MappedAnnotationField ->
FieldParsers.unpackSource(field.field, source).let { sourceElement: SourceElement -> FieldParsers.unpackSource(field.key, field.field, source).let { sourceElement: SourceElement ->
when (sourceElement) { when (sourceElement) {
is SimpleString -> subject.addRicoCarrierType(listOf(field.toLiteral(sourceElement.value))) is SimpleString -> subject.addRicoCarrierType(listOf(field.toLiteral(sourceElement.value)))
is StringList -> is StringList ->
......
...@@ -30,7 +30,7 @@ import org.apache.logging.log4j.LogManager ...@@ -30,7 +30,7 @@ import org.apache.logging.log4j.LogManager
class DateFieldMapper(private val directMapField: DirectMapField) : AbstractFieldMapper() { class DateFieldMapper(private val directMapField: DirectMapField) : AbstractFieldMapper() {
private val log = LogManager.getLogger(this::class.java) private val log = LogManager.getLogger(this::class.java)
override fun apply(source: Map<String, Any>, subject: IResource) { override fun apply(source: Map<String, Any>, subject: IResource) {
FieldParsers.unpackSource(directMapField.field, source).let { sourceElement: SourceElement -> FieldParsers.unpackSource(directMapField.key, directMapField.field, source).let { sourceElement: SourceElement ->
when (sourceElement) { when (sourceElement) {
is SimpleString -> subject.addDate(directMapField.key, sourceElement.value) is SimpleString -> subject.addDate(directMapField.key, sourceElement.value)
is StringList -> sourceElement.value.forEach { is StringList -> sourceElement.value.forEach {
......
...@@ -30,7 +30,7 @@ import org.apache.logging.log4j.LogManager ...@@ -30,7 +30,7 @@ import org.apache.logging.log4j.LogManager
class DirectFieldMapper(private val directMapField: DirectMapField) : AbstractFieldMapper() { class DirectFieldMapper(private val directMapField: DirectMapField) : AbstractFieldMapper() {
private val log = LogManager.getLogger(this::class.java) private val log = LogManager.getLogger(this::class.java)
override fun apply(source: Map<String, Any>, subject: IResource) { override fun apply(source: Map<String, Any>, subject: IResource) {
FieldParsers.unpackSource(directMapField.field, source).let { sourceElement: SourceElement -> FieldParsers.unpackSource(directMapField.key, directMapField.field, source).let { sourceElement: SourceElement ->
when (sourceElement) { when (sourceElement) {
is SimpleString -> subject.addLiteral(directMapField.key, directMapField.toLiteral(sourceElement.value)) is SimpleString -> subject.addLiteral(directMapField.key, directMapField.toLiteral(sourceElement.value))
is StringList -> sourceElement.value.forEach { is StringList -> sourceElement.value.forEach {
......
...@@ -43,7 +43,7 @@ class ExpandedRuleFieldMapper( ...@@ -43,7 +43,7 @@ class ExpandedRuleFieldMapper(
configField.forEach { field -> configField.forEach { field ->
when (field) { when (field) {
is MappedAnnotationField -> is MappedAnnotationField ->
FieldParsers.unpackSource(field.field, source).let { sourceElement: SourceElement -> FieldParsers.unpackSource(field.key, field.field, source).let { sourceElement: SourceElement ->
when (sourceElement) { when (sourceElement) {
is SimpleString -> is SimpleString ->
if (properties.size == 1) if (properties.size == 1)
......
...@@ -30,7 +30,7 @@ import org.apache.logging.log4j.LogManager ...@@ -30,7 +30,7 @@ import org.apache.logging.log4j.LogManager
class PrefixFieldMapper(private val prefixField: PrefixField) : AbstractFieldMapper() { class PrefixFieldMapper(private val prefixField: PrefixField) : AbstractFieldMapper() {
private val log = LogManager.getLogger(this::class.java) private val log = LogManager.getLogger(this::class.java)
override fun apply(source: Map<String, Any>, subject: IResource) { override fun apply(source: Map<String, Any>, subject: IResource) {
FieldParsers.unpackSource(prefixField.field, source).let { sourceElement: SourceElement -> FieldParsers.unpackSource(prefixField.key, prefixField.field, source).let { sourceElement: SourceElement ->
when (sourceElement) { when (sourceElement) {
is SimpleString -> subject.addLiteral(prefixField.key, prefixField.toLiteral(sourceElement.value)) is SimpleString -> subject.addLiteral(prefixField.key, prefixField.toLiteral(sourceElement.value))
is StringList -> sourceElement.value.forEach { is StringList -> sourceElement.value.forEach {
......
...@@ -36,7 +36,7 @@ class RicoConceptMapper(private val rdfType: String, val field: AnnotationField) ...@@ -36,7 +36,7 @@ class RicoConceptMapper(private val rdfType: String, val field: AnnotationField)
override fun apply(source: Map<String, Any>, subject: IResource) { override fun apply(source: Map<String, Any>, subject: IResource) {
when (field) { when (field) {
is MappedAnnotationField -> is MappedAnnotationField ->
FieldParsers.unpackSource(field.field, source).let { FieldParsers.unpackSource(field.key, field.field, source).let {
when (it) { when (it) {
is SimpleString -> subject.addRicoConcept(rdfType, field.key, listOf(field.toLiteral(it.value))) is SimpleString -> subject.addRicoConcept(rdfType, field.key, listOf(field.toLiteral(it.value)))
is StringList -> is StringList ->
......
...@@ -36,7 +36,7 @@ class RuleFieldMapper(private val configField: ConfigField) : AbstractFieldMappe ...@@ -36,7 +36,7 @@ class RuleFieldMapper(private val configField: ConfigField) : AbstractFieldMappe
override fun apply(source: Map<String, Any>, subject: IResource) { override fun apply(source: Map<String, Any>, subject: IResource) {
when (configField) { when (configField) {
is MappedAnnotationField -> is MappedAnnotationField ->
FieldParsers.unpackSource(configField.field, source).let { FieldParsers.unpackSource(configField.key, configField.field, source).let {
when (it) { when (it) {
is SimpleString -> subject.addRule( is SimpleString -> subject.addRule(
configField.key, configField.key,
......
...@@ -44,7 +44,7 @@ abstract class TypeFieldMapper : AbstractFieldMapper() { ...@@ -44,7 +44,7 @@ abstract class TypeFieldMapper : AbstractFieldMapper() {
} }
private fun addMappedField(field: MappedAnnotationField, source: Map<String, Any>) { private fun addMappedField(field: MappedAnnotationField, source: Map<String, Any>) {
FieldParsers.unpackSource(field.field, source).let { sourceElement -> FieldParsers.unpackSource(field.key, field.field, source).let { sourceElement ->
when (sourceElement) { when (sourceElement) {
is SimpleString -> is SimpleString ->
if (properties.size == 1) { if (properties.size == 1) {
...@@ -55,7 +55,7 @@ abstract class TypeFieldMapper : AbstractFieldMapper() { ...@@ -55,7 +55,7 @@ abstract class TypeFieldMapper : AbstractFieldMapper() {
is StringList -> is StringList ->
sourceElement.value.forEachIndexed { index, s -> sourceElement.value.forEachIndexed { index, s ->
if (s.isNotEmpty()) { if (s.isNotEmpty()) {
if (properties.size <= index + 1) { if (properties.size >= index + 1) {
properties[index].add(Pair(field.key, field.toLiteral(s))) properties[index].add(Pair(field.key, field.toLiteral(s)))
} else { } else {
properties.add(index, mutableListOf(Pair(field.key, field.toLiteral(s)))) properties.add(index, mutableListOf(Pair(field.key, field.toLiteral(s))))
......
...@@ -19,12 +19,18 @@ ...@@ -19,12 +19,18 @@
package ch.memobase.test package ch.memobase.test
import ch.memobase.builder.Record import ch.memobase.builder.Record
import ch.memobase.builder.ResourceBuilder
import ch.memobase.mapping.MapperParsers import ch.memobase.mapping.MapperParsers
import ch.memobase.mapping.MappingConfigurationParser
import ch.memobase.rdf.RICO import ch.memobase.rdf.RICO
import ch.memobase.settings.HeaderMetadata
import com.beust.klaxon.Klaxon
import java.io.File
import java.io.FileOutputStream import java.io.FileOutputStream
import org.apache.jena.rdf.model.impl.SelectorImpl import org.apache.jena.rdf.model.impl.SelectorImpl
import org.apache.jena.riot.Lang import org.apache.jena.riot.Lang
import org.apache.jena.riot.RDFDataMgr import org.apache.jena.riot.RDFDataMgr
import org.apache.jena.riot.RDFFormat
import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Test import org.junit.jupiter.api.Test
import org.junit.jupiter.api.TestInstance import org.junit.jupiter.api.TestInstance
...@@ -33,6 +39,18 @@ import org.junit.jupiter.api.assertAll ...@@ -33,6 +39,18 @@ import org.junit.jupiter.api.assertAll
@TestInstance(TestInstance.Lifecycle.PER_CLASS) @TestInstance(TestInstance.Lifecycle.PER_CLASS)
class TestAgentMapper { class TestAgentMapper {
private val klaxon = Klaxon()
private val headerMetadata = HeaderMetadata(
"test-001",
"1",
"test",
false,
"record",
"identifierMain",
0, 0, 0, 0
)
private val mapper = MapperParsers.buildAgentMapper( private val mapper = MapperParsers.buildAgentMapper(
"creators", "creators",
listOf( listOf(
...@@ -47,7 +65,35 @@ class TestAgentMapper { ...@@ -47,7 +65,35 @@ class TestAgentMapper {
) )
)[0] )[0]
private val regex = Regex("(_:B[A-Za-z0-9]+)")
private val regexTime = Regex("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}")
private fun sort(source: List<String>): String {
return source.map {
var replacedString = it
for (matchResult in regex.findAll(it)) {
replacedString = replacedString.replace(matchResult.groups[0]?.value.orEmpty(), "_:B")
}
for (matchResult in regexTime.findAll(it)) {
replacedString = replacedString.replace(matchResult.groups[0]?.value.orEmpty(), "2020-10-10T09:10:22")
}
replacedString
}.sorted().reduce { acc, s -> acc + "\n" + s }.trim()
}
private val resourcePath = "src/test/resources/turtleOutput" private val resourcePath = "src/test/resources/turtleOutput"
private val inputPath = "src/test/resources/agent-mapper"
private fun readInputFile(fileName: String): String {
return File("$inputPath/$fileName.json").readText()
}
private fun readOutputFile(fileName: String): String {
return sort(File("$inputPath/$fileName.nt").readLines())
}
private fun readMapping(fileName: String): ByteArray {
return File("$inputPath/$fileName.yml").readBytes()
}
// TODO: Actually test this. // TODO: Actually test this.
@Test @Test
...@@ -155,4 +201,36 @@ class TestAgentMapper { ...@@ -155,4 +201,36 @@ class TestAgentMapper {
Lang.TURTLE Lang.TURTLE
) )
} }
@Test
fun `test multiple contributor names`() {
val source = klaxon.parse<Map<String, Any>>(readInputFile("input-multiple-contributors")).orEmpty()
val mapping = MappingConfigurationParser(readMapping("mapping-multiple-contributors"))
val configuration = mapping.get()
val result = ResourceBuilder(
source,
configuration,
headerMetadata.institutionId,
headerMetadata.recordSetId,
headerMetadata.isPublished
).extractRecordId()
.extractRecordTypeValue()
.generateRecord()
.generatePhysicalObject()
.generateDigitalObject()
.addDerivedConnection()
val ntriples = result
.writeRecord(RDFFormat.NTRIPLES_UTF8)
val turtle = result.writeRecord(RDFFormat.TURTLE_PRETTY)
assertThat(sort(ntriples.second.split("\n")))
.isEqualTo(readOutputFile("output-multiple-contributors"))
FileOutputStream(File("$resourcePath/output-multiple-contributors.ttl")).use {
it.bufferedWriter().use { writer ->
writer.write(turtle.second)
}
}
}
} }
{
"id": "identifier",
"contributorPerson": [
{
"name": "Simon Epiney / Conseiller national PDC Valais",
"role": "Gesprächspartner"
},
{
"name": "Markus Ruf / Conseiller national Démocrates suisses Berne",
"role": "Gesprächspartner"
},
{
"name": "Jean Cavadini / Conseiller aux Etats PL Neuchâtel",
"role": "Gesprächspartner"
},
{
"name": "Irène Gardiol / Conseillère nationale écologiste Vaud",
"role": "Gesprächspartner"
},
{
"name": "Flavio Maspoli / Conseiller national Lega Tessin",
"role": "Gesprächspartner"
},
{
"name": "Flavio Cotti / Conseiller fédéral DFAE",
"role": "Gesprächspartner"
},
{
"name": "Suzette Sandoz / Conseillère nationale PL Vaud",
"role": "Gesprächspartner"
},
{
"name": "Matthias Krafft / Directeur Direction droit international public DFAE",
"role": "Gesprächspartner"
},
{
"name": "Vital Darbellay / Conseiller national PDC Valais",
"role": "Gesprächspartner"
},
{
"name": "Jacques Martin / Conseiller aux Etats PRD Vaud",
"role": "Gesprächspartner"
},
{
"name": "Pierre-André Tschanz / Journaliste RSI"
},
{
"name": "Pierre-André Tschanz / Journaliste RSI",
"role": "Interviewer"
},
{
"name": "Catherine Miskiewicz / Journaliste RSI"
},
{
"name": "Catherine Miskiewicz",
"role": "ModeratorIn"
},
{
"name": "Pierre-André Tschanz",
"role": "ModeratorIn"
},
{
"name": "Pierre-André Tschanz",
"role": "Autor"
},
{
"name": "Catherine Miskiewicz",
"role": "Autor"
}
]
}
\ No newline at end of file
record:
uri: id
type:
const: Foto
contributors:
person:
name: contributorPerson.name
relationName: contributorPerson.role
\ No newline at end of file
...@@ -5,15 +5,6 @@ ...@@ -5,15 +5,6 @@
@prefix ebucore: <http://www.ebu.ch/metadata/ontologies/ebucore/ebucore#> . @prefix ebucore: <http://www.ebu.ch/metadata/ontologies/ebucore/ebucore#> .
@prefix skos: <http://www.w3.org/2004/02/skos/core#> . @prefix skos: <http://www.w3.org/2004/02/skos/core#> .
_:b0 a rico:CreationRelation ;
rico:creationRelationHasSource <https://memobase.ch/record/rs1-1> ;
rico:creationRelationHasTarget [ a rico:Person ;
rico:agentIsTargetOfCreationRelation
_:b0 ;
rico:name "Hans Zimmer"
] ;
rico:type "creator" .
<https://memobase.ch/record/rs1-1> <https://memobase.ch/record/rs1-1>
a rico:Record ; a rico:Record ;
rdau:P60451 <https://memobase.ch/institution/mrv> ; rdau:P60451 <https://memobase.ch/institution/mrv> ;
...@@ -28,3 +19,12 @@ _:b0 a rico:CreationRelation ; ...@@ -28,3 +19,12 @@ _:b0 a rico:CreationRelation ;
rico:recordResourceOrInstantiationIsSourceOfCreationRelation rico:recordResourceOrInstantiationIsSourceOfCreationRelation
_:b0 ; _:b0 ;
rico:type "Foto" . rico:type "Foto" .
_:b0 a rico:CreationRelation ;
rico:creationRelationHasSource <https://memobase.ch/record/rs1-1> ;
rico:creationRelationHasTarget [ a rico:Person ;
rico:agentIsTargetOfCreationRelation
_:b0 ;
rico:name "Hans Zimmer"
] ;
rico:type "creator" .
...@@ -5,16 +5,6 @@ ...@@ -5,16 +5,6 @@
@prefix ebucore: <http://www.ebu.ch/metadata/ontologies/ebucore/ebucore#> . @prefix ebucore: <http://www.ebu.ch/metadata/ontologies/ebucore/ebucore#> .
@prefix skos: <http://www.w3.org/2004/02/skos/core#> . @prefix skos: <http://www.w3.org/2004/02/skos/core#> .