Commit c2affc40 authored by Jonas Waeber's avatar Jonas Waeber
Browse files

Refactor config fields

It is now easier to expand the system and easier to not miss something! Expands the options for most source fields.
parent 28c0d690
Pipeline #8914 failed with stages
in 6 minutes and 9 seconds
......@@ -21,8 +21,7 @@ package org.memobase.builder
import org.apache.jena.rdf.model.Literal
interface IResource {
fun addStringLiteral(property: String, value: String)
fun addLangLiteral(property: String, value: String, tag: String)
fun addLiteral(property: String, value: Literal)
fun addLanguage(type: String, text: String)
fun addTitle(type: String, text: String, language: String)
fun addIdentifier(type: String, identifier: String)
......
......@@ -50,14 +50,10 @@ abstract class RecordResource(institutionId: String) : IResource {
protected fun addRdfType(type: Resource) {
resource.addProperty(RDF.type, type)
}
override fun addStringLiteral(property: String, value: String) {
override fun addLiteral(property: String, value: Literal) {
resource.addLiteral(KEYS.keysToPropertyMap[property], value)
}
override fun addLangLiteral(property: String, value: String, tag: String) {
resource.addLiteral(KEYS.keysToPropertyMap[property], model.createLiteral(value, tag))
}
override fun addLanguage(type: String, text: String) {
val blank = model.createResource()
blank.addProperty(RICO.name, literal(text))
......
......@@ -20,12 +20,14 @@ package org.memobase.mapping
import org.apache.logging.log4j.LogManager
import org.memobase.mapping.exceptions.InvalidMappingException
import org.memobase.mapping.fields.AnnotationField
import org.memobase.mapping.fields.ConfigField
import org.memobase.mapping.fields.ConstField
import org.memobase.mapping.fields.ConstantField
import org.memobase.mapping.fields.DirectMapField
import org.memobase.mapping.fields.LanguageField
import org.memobase.mapping.fields.ListField
import org.memobase.mapping.fields.PrefixField
import org.memobase.mapping.fields.SimpleField
import org.memobase.mapping.fields.SimpleAnnotationField
import org.memobase.mapping.mappers.AgentFieldMapper
import org.memobase.mapping.mappers.ConstantFieldMapper
import org.memobase.mapping.mappers.DateFieldMapper
......@@ -38,7 +40,7 @@ import org.memobase.mapping.mappers.PlaceFieldMapper
import org.memobase.mapping.mappers.PrefixFieldMapper
import org.memobase.mapping.mappers.RuleFieldMapper
import org.memobase.mapping.mappers.SkosConceptFieldMapper
import org.memobase.mapping.mappers.SimpleFieldMapper
import org.memobase.mapping.mappers.DirectFieldMapper
import org.memobase.mapping.mappers.TitleFieldMapper
import org.memobase.mapping.mappers.TypeFieldMapper
import org.memobase.rdf.SKOS
......@@ -112,7 +114,7 @@ class MappingConfig(directory: String) {
// literal properties
KEYS.name, KEYS.title, KEYS.descriptiveNote, KEYS.scopeAndContent, KEYS.sameAs,
KEYS.abstract, KEYS.source, KEYS.hasSponsoringAgent, KEYS.hasFindingAid ->
recordFieldMappers.add(buildDirectMappers(entry))
recordFieldMappers.add(buildAnnotationMappers(entry))
KEYS.rights ->
recordFieldMappers.addAll(buildRuleMappers(entry.value))
KEYS.titles ->
......@@ -124,7 +126,7 @@ class MappingConfig(directory: String) {
KEYS.titles,
titleEntry as Map.Entry<String, Any>, titleTypes
)
if (configField is LanguageField) {
if (configField is ConfigField.AnnotationField.ComplexAnnotationField.LanguageField) {
recordFieldMappers.add(
TitleFieldMapper(
configField
......@@ -177,8 +179,8 @@ class MappingConfig(directory: String) {
KEYS.placeOfCapture, KEYS.relatedPlaces ->
recordFieldMappers.addAll(buildPlaceMapper(key, entry.value))
KEYS.creationDate, KEYS.issuedDate, KEYS.temporal ->
when (val value = parseField(entry)) {
is SimpleField -> recordFieldMappers.add(
when (val value = parseAnnotationField(entry)) {
is DirectMapField -> recordFieldMappers.add(
DateFieldMapper(value)
)
else -> throw InvalidMappingException("Dates only allow simple field assignment. Invalid mapping ${entry.key}.")
......@@ -196,7 +198,7 @@ class MappingConfig(directory: String) {
KEYS.descriptiveNote, KEYS.medium, KEYS.physicalCharacteristics,
KEYS.colour, KEYS.duration, KEYS.displayAspectRatio, KEYS.audioTrackConfiguration,
KEYS.playbackSpeed, KEYS.hasStandard ->
physicalObjectFieldMappers.add(buildDirectMappers(entry))
physicalObjectFieldMappers.add(buildAnnotationMappers(entry))
KEYS.identifiers ->
physicalObjectFieldMappers.addAll(buildIdentifierMapper(entry.value))
KEYS.rights ->
......@@ -210,7 +212,7 @@ class MappingConfig(directory: String) {
for (entry in source) {
when (val key = entry.key) {
KEYS.locator, KEYS.descriptiveNote, KEYS.duration ->
digitalObjectFieldMappers.add(buildDirectMappers(entry))
digitalObjectFieldMappers.add(buildAnnotationMappers(entry))
KEYS.rights ->
digitalObjectFieldMappers.addAll(buildRuleMappers(entry.value))
else -> throw InvalidMappingException("Unknown key '$key' in digital object mapping.")
......@@ -218,14 +220,13 @@ class MappingConfig(directory: String) {
}
}
private fun buildDirectMappers(value: Any?): IFieldMapper {
return when (val field = parseField(value as Map.Entry<String, Any>)) {
is SimpleField -> SimpleFieldMapper(field)
is ConstField -> ConstantFieldMapper(field)
private fun buildAnnotationMappers(value: Any?): IFieldMapper {
return when (val field = parseAnnotationField(value as Map.Entry<String, Any>)) {
is DirectMapField -> DirectFieldMapper(field)
is ConstantField -> ConstantFieldMapper(field)
is ListField -> ListFieldMapper(field)
is LanguageField -> LanguageFieldMapper(field)
is PrefixField -> PrefixFieldMapper(field)
else -> throw InvalidMappingException("This can't happen since there is no other class of this type...")
}
}
......@@ -302,7 +303,7 @@ class MappingConfig(directory: String) {
(mapper as AgentFieldMapper).property = key
if (value.containsKey(KEYS.creatorType)) {
mapper.creationType = parseField(value.entries.first { i -> i.key == KEYS.creatorType })
mapper.creationType = parseAnnotationField(value.entries.first { i -> i.key == KEYS.creatorType })
}
return mapper
}
......@@ -355,7 +356,7 @@ class MappingConfig(directory: String) {
KEYS.languages,
subEntry, languageTypes
)
if (configField is SimpleField) {
if (configField is DirectMapField) {
recordFieldMappers.add(LanguageMapper(configField))
} else
throw InvalidMappingException("Languages only allow simple direct mappings.")
......@@ -369,42 +370,89 @@ class MappingConfig(directory: String) {
KEYS.identifiers,
it, identifierTypes
)
if (configField is SimpleField) {
if (configField is DirectMapField) {
return@map IdentifierFieldMapper(configField)
} else
throw InvalidMappingException("Identifiers only allow simple direct mappings.")
}
}
private fun parseField(entry: Map.Entry<String, Any>): ConfigField {
private fun parsePrefixField(key: String, values: Map<String, String>): PrefixField {
return values["field"].let { field ->
if (field != null) {
values["value"].let { value ->
if (value != null) {
PrefixField(key, value, field)
} else {
throw InvalidMappingException("Section: $key. A 'value' subfield in prefix definition is required.")
}
}
} else {
throw InvalidMappingException("Section: $key. A 'field' subfield in prefix definition is required.")
}
}
}
private fun parseSimpleAnnotationField(key: String, item: Any): SimpleAnnotationField {
return when (item) {
is String -> DirectMapField(key, item)
is Map<*, *> -> {
when {
item.containsKey("const") -> {
ConstantField(key, item["const"] as String)
}
item.containsKey("prefix") -> {
try {
parsePrefixField(key, item["prefix"] as Map<String, String>)
} catch (ex: ClassCastException) {
throw InvalidMappingException("Invalid structure for prefix field in section $key.")
}
}
else -> {
throw InvalidMappingException("Invalid key for a simple annotation field in section $key. Expected 'const' or 'prefix'.")
}
}
}
else -> throw InvalidMappingException("Invalid value for a simple annotation field in section $key. Expected 'const' or 'prefix' or a direct map.")
}
}
private fun parseFieldList(key: String, items: List<Any>): List<SimpleAnnotationField> {
return items.map {
parseSimpleAnnotationField(key, it)
}
}
private fun parseAnnotationField(entry: Map.Entry<String, Any>): AnnotationField {
when (val value = entry.value) {
is String -> return SimpleField(entry.key, value)
is String -> return DirectMapField(entry.key, value)
is Map<*, *> -> {
return if (value.containsKey("const")) {
ConstField(entry.key, value["const"] as String)
ConstantField(entry.key, value["const"] as String)
} else if (value.containsKey("prefix")) {
value["prefix"].let {
if (it is Map<*, *>) {
if (it.containsKey("field") && it.containsKey("prefix")) {
PrefixField(entry.key, it["prefix"] as String, it["field"] as String)
} else {
throw InvalidMappingException("The prefix field under ${entry.key} expected a map with the keys 'prefix' and 'field'.")
}
} else {
throw InvalidMappingException("The prefix field expects a mapping inside. Found $it instead.")
}
try {
parsePrefixField(entry.key, value["prefix"] as Map<String, String>)
} catch (ex: ClassCastException) {
throw InvalidMappingException("Invalid structure for prefix field in section ${entry.key}.")
}
} else {
val mutableList = mutableListOf<Pair<String, List<String>>>()
val mutableList = mutableListOf<Pair<String, List<SimpleAnnotationField>>>()
for (k in value.keys) {
if (k is String && listOf("de", "it", "fr").contains(k)) {
when (val source = value[k]) {
is String -> mutableList.add(Pair(k, listOf(source)))
is List<*> -> mutableList.add(Pair(k, source as List<String>))
else -> throw InvalidMappingException(
"Could not parse content of language tag definition. Must be list or string."
)
val resultAnnotations = when (val items = value[k]) {
is List<*> -> {
parseFieldList(entry.key, items as List<Any>)
}
is Any -> listOf(parseSimpleAnnotationField(entry.key, items))
else -> throw InvalidMappingException("Language map definition may not be empty.")
}
if (resultAnnotations.isEmpty()) {
throw InvalidMappingException("No valid definitions found under language tag $k in section ${entry.key}")
}
mutableList.add(Pair(k, resultAnnotations))
}
}
if (mutableList.isNotEmpty()) {
......@@ -416,7 +464,7 @@ class MappingConfig(directory: String) {
}
is List<*> -> return ListField(
entry.key,
value as List<String>
parseFieldList(entry.key, value as List<Any>)
)
else -> throw InvalidMappingException("Unknown structure for field mapping: $entry")
}
......@@ -428,7 +476,7 @@ class MappingConfig(directory: String) {
Map: List<String>
): ConfigField {
if (Map.contains(entry.key)) {
return parseField(entry)
return parseAnnotationField(entry)
} else {
throw InvalidMappingException("The section $parent does not allow the key: ${entry.key}. Use any of $Map.")
}
......
......@@ -18,4 +18,96 @@
package org.memobase.mapping.fields
open class ConfigField
\ No newline at end of file
import org.apache.jena.rdf.model.Literal
import org.apache.jena.rdf.model.ResourceFactory
sealed class ConfigField {
sealed class AnnotationField : ConfigField() {
sealed class SimpleAnnotationField : AnnotationField() {
data class ConstantField(val key: String, val constant: String) : SimpleAnnotationField() {
fun toLiteral(): Literal {
return literal(constant)
}
fun toLangLiteral(tag: String): Literal {
return langLiteral(constant, tag)
}
}
sealed class MappedAnnotationField(val key: String, val field: String) : SimpleAnnotationField() {
data class PrefixField(
private val prefixKey: String,
val prefix: String,
private val prefixField: String
) : MappedAnnotationField(prefixKey, prefixField) {
override fun toLiteral(value: String): Literal {
return literal(prefix + value.trim())
}
override fun toLangLiteral(value: String, tag: String): Literal {
return langLiteral(prefix + value.trim(), tag)
}
}
data class DirectMapField(private val directKey: String, private val directField: String) :
MappedAnnotationField(directKey, directField) {
override fun toLiteral(value: String): Literal {
return literal(value)
}
override fun toLangLiteral(value: String, tag: String): Literal {
return langLiteral(value, tag)
}
}
abstract fun toLiteral(value: String): Literal
abstract fun toLangLiteral(value: String, tag: String): Literal
}
}
sealed class ComplexAnnotationField : AnnotationField() {
data class ListField(val key: String, val fields: List<SimpleAnnotationField>) : ComplexAnnotationField() {
fun toLiterals(value: String): List<Literal> {
return fields.map {
when (it) {
is MappedAnnotationField -> it.toLiteral(value)
is ConstantField -> it.toLiteral()
}
}
}
}
data class LanguageField(val key: String, val fields: List<Pair<String, List<SimpleAnnotationField>>>) :
ComplexAnnotationField() {
fun toLangLiterals(value: String): List<Literal> {
return fields.map { pairs ->
return pairs.second.map {
when (it) {
is MappedAnnotationField -> it.toLangLiteral(value, pairs.first)
is ConstantField -> it.toLangLiteral(pairs.first)
}
}
}
}
}
}
}
protected fun langLiteral(text: String, language: String): Literal =
ResourceFactory.createLangLiteral(text.trim(), language)
protected fun literal(text: String): Literal = ResourceFactory.createStringLiteral(text.trim())
}
typealias AnnotationField = ConfigField.AnnotationField
typealias DirectMapField = ConfigField.AnnotationField.SimpleAnnotationField.MappedAnnotationField.DirectMapField
typealias PrefixField = ConfigField.AnnotationField.SimpleAnnotationField.MappedAnnotationField.PrefixField
typealias ConstantField = ConfigField.AnnotationField.SimpleAnnotationField.ConstantField
typealias SimpleAnnotationField = ConfigField.AnnotationField.SimpleAnnotationField
typealias MappedAnnotationField = ConfigField.AnnotationField.SimpleAnnotationField.MappedAnnotationField
typealias ListField = ConfigField.AnnotationField.ComplexAnnotationField.ListField
typealias LanguageField = ConfigField.AnnotationField.ComplexAnnotationField.LanguageField
typealias ComplexAnnotationField = ConfigField.AnnotationField.ComplexAnnotationField
\ No newline at end of file
/*
* mapper-service
* Copyright (C) 2020 Memoriav
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.memobase.mapping.fields
data class ConstField(val key: String, val constant: String) : ConfigField()
\ No newline at end of file
/*
* mapper-service
* Copyright (C) 2020 Memoriav
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.memobase.mapping.fields
data class LanguageField(val key: String, val fields: List<Pair<String, List<String>>>) : ConfigField()
\ No newline at end of file
/*
* mapper-service
* Copyright (C) 2020 Memoriav
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.memobase.mapping.fields
data class ListField(val key: String, val fields: List<String>) : ConfigField()
\ No newline at end of file
/*
* mapper-service
* Copyright (C) 2020 Memoriav
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.memobase.mapping.fields
data class PrefixField(val key: String, val prefix: String, val field: String) : ConfigField()
\ No newline at end of file
/*
* mapper-service
* Copyright (C) 2020 Memoriav
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.memobase.mapping.fields
data class SimpleField(val key: String, val field: String) : ConfigField()
\ No newline at end of file
......@@ -20,8 +20,8 @@ package org.memobase.mapping.mappers
import org.memobase.builder.IResource
import org.memobase.mapping.fields.ConfigField
import org.memobase.mapping.fields.ConstField
import org.memobase.mapping.fields.SimpleField
import org.memobase.mapping.fields.ConstantField
import org.memobase.mapping.fields.MappedAnnotationField
class AgentFieldMapper : TypeFieldMapper() {
var creationType: ConfigField? = null
......@@ -30,12 +30,15 @@ class AgentFieldMapper : TypeFieldMapper() {
translateProperties(source, subject)
if (properties.size > 0) {
when (val creationTypeField = creationType) {
is ConstField -> subject.addCreationRelation(creationTypeField.constant, type, properties)
is SimpleField -> source[creationTypeField.key].let {
if (it != null) subject.addCreationRelation(it, type, properties) }
is ConstantField -> subject.addCreationRelation(creationTypeField.constant, type, properties)
is MappedAnnotationField ->
source[creationTypeField.key].let {
if (it != null) {
subject.addCreationRelation(it, type, properties)
}
}
else -> subject.addAgent(property, type, properties)
}
}
}
}
\ No newline at end of file
......@@ -19,10 +19,10 @@
package org.memobase.mapping.mappers
import org.memobase.builder.IResource
import org.memobase.mapping.fields.ConstField
import org.memobase.mapping.fields.ConstantField
class ConstantFieldMapper(private val constantField: ConstField) : IFieldMapper {
class ConstantFieldMapper(private val constantField: ConstantField) : IFieldMapper {
override fun apply(source: Map<String, String>, subject: IResource) {
subject.addStringLiteral(constantField.key, constantField.constant)
subject.addLiteral(constantField.key, constantField.toLiteral())
}
}
\ No newline at end of file
......@@ -19,13 +19,13 @@
package org.memobase.mapping.mappers
import org.memobase.builder.IResource
import org.memobase.mapping.fields.SimpleField
import org.memobase.mapping.fields.DirectMapField
class DateFieldMapper(private val simpleField: SimpleField) : IFieldMapper {
class DateFieldMapper(private val directMapField: DirectMapField) : IFieldMapper {
override fun apply(source: Map<String, String>, subject: IResource) {
source[simpleField.field].let {
source[directMapField.field].let {
if (it != null) {
subject.addDate(simpleField.key, it)
subject.addDate(directMapField.key, it)
}
}
}
......
......@@ -19,13 +19,13 @@
package org.memobase.mapping.mappers
import org.memobase.builder.IResource
import org.memobase.mapping.fields.SimpleField
import org.memobase.mapping.fields.DirectMapField
class SimpleFieldMapper(private val simpleField: SimpleField) : IFieldMapper {
class DirectFieldMapper(private val directMapField: DirectMapField) : IFieldMapper {
override fun apply(source: Map<String, String>, subject: IResource) {
source[simpleField.field].let {
source[directMapField.field].let {
if (it != null) {
subject.addStringLiteral(simpleField.key, it)
subject.addLiteral(directMapField.key, directMapField.toLiteral(it))
}
}
}
......
......@@ -19,12 +19,12 @@
package org.memobase.mapping.mappers
import org.memobase.builder.IResource
import org.memobase.mapping.fields.SimpleField
import org.memobase.mapping.fields.DirectMapField
class IdentifierFieldMapper(private val simpleField: SimpleField) : IFieldMapper {
class IdentifierFieldMapper(private val directMapField: DirectMapField) : IFieldMapper {
override fun apply(source: Map<String, String>, subject: IResource) {
if (source.containsKey(simpleField.field)) {
subject.addIdentifier(simpleField.key, source[simpleField.field] as String)
if (source.containsKey(directMapField.field)) {
subject.addIdentifier(directMapField.key, source[directMapField.field] as String)
}
}
}
\ No newline at end of file