In order to mitigate against the brute force attacks against Gitlab accounts, we are moving to all edu-ID Logins. We would like to remind you to link your account with your edu-id. Login will be possible only by edu-ID after November 30, 2021. Here you can find the instructions for linking your account.

If you don't have a SWITCH edu-ID, you can create one with this guide here

kind regards

Commit c32da60c authored by Jonas Waeber's avatar Jonas Waeber
Browse files

Refactor constants and stuff

parent ec4a54d4
...@@ -21,8 +21,23 @@ package org.memobase ...@@ -21,8 +21,23 @@ package org.memobase
import ch.memobase.rdf.NS import ch.memobase.rdf.NS
import com.beust.klaxon.JsonObject import com.beust.klaxon.JsonObject
import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.LogManager
import org.memobase.builders.* import org.memobase.builders.AgentContainerBuilder
import org.memobase.helpers.* import org.memobase.builders.DateContainerBuilder
import org.memobase.builders.EnrichedFacetContainerBuilder
import org.memobase.builders.FacettedContainerBuilder
import org.memobase.builders.IFieldBuilder
import org.memobase.builders.PersonFacetBuilder
import org.memobase.builders.PlaceFacetBuilder
import org.memobase.builders.SuggestContainerBuilder
import org.memobase.helpers.AspectRatio
import org.memobase.helpers.DocumentTypeMapper
import org.memobase.helpers.Extract
import org.memobase.helpers.FacetBuildHelpers
import org.memobase.helpers.Filter
import org.memobase.helpers.InstitutionAndRecordSetExtractionHelper.extractInstitution
import org.memobase.helpers.InstitutionAndRecordSetExtractionHelper.extractRecordSet
import org.memobase.helpers.KEYS
import org.memobase.helpers.ReuseStatementMap
import org.memobase.model.DocumentsSearchDoc import org.memobase.model.DocumentsSearchDoc
import org.memobase.model.EnrichedDigitalMetadata import org.memobase.model.EnrichedDigitalMetadata
import org.memobase.model.LanguageContainer import org.memobase.model.LanguageContainer
...@@ -201,8 +216,8 @@ class DocumentsSearchDocBuilder( ...@@ -201,8 +216,8 @@ class DocumentsSearchDocBuilder(
sameAs = Extract.listOfStrings(record["sameAs"]), sameAs = Extract.listOfStrings(record["sameAs"]),
abstract = Extract.languageContainer("abstract", record["abstract"]), abstract = Extract.languageContainer("abstract", record["abstract"]),
recordId = id, recordId = id,
institution = Meta.extractInstitution(record), institution = extractInstitution(record),
recordSet = Meta.extractRecordSet(record), recordSet = extractRecordSet(record),
descriptiveNote = Extract.languageContainer("descriptiveNote", record["descriptiveNote"]), descriptiveNote = Extract.languageContainer("descriptiveNote", record["descriptiveNote"]),
scopeAndContent = Extract.languageContainer("scopeAndContent", record["scopeAndContent"]), scopeAndContent = Extract.languageContainer("scopeAndContent", record["scopeAndContent"]),
relatedMaterial = Extract.languageContainer("relation", record["relation"]), relatedMaterial = Extract.languageContainer("relation", record["relation"]),
......
...@@ -33,7 +33,6 @@ import org.memobase.helpers.DocumentTypeMapper ...@@ -33,7 +33,6 @@ import org.memobase.helpers.DocumentTypeMapper
import org.memobase.helpers.ElasticSearchWrapper import org.memobase.helpers.ElasticSearchWrapper
import org.memobase.helpers.InstitutionTypeMapper import org.memobase.helpers.InstitutionTypeMapper
import org.memobase.helpers.JSON import org.memobase.helpers.JSON
import org.memobase.helpers.KEYS
import org.memobase.helpers.KEYS.SettingsProps import org.memobase.helpers.KEYS.SettingsProps
import org.memobase.model.DocumentsSearchDoc import org.memobase.model.DocumentsSearchDoc
import org.memobase.model.InstitutionSearchDoc import org.memobase.model.InstitutionSearchDoc
...@@ -69,9 +68,9 @@ class KafkaTopology(private val settings: SettingsLoader) { ...@@ -69,9 +68,9 @@ class KafkaTopology(private val settings: SettingsLoader) {
.filter { _, value -> value.isNotEmpty() } .filter { _, value -> value.isNotEmpty() }
.mapValues { value -> JSON.unpack(value) } .mapValues { value -> JSON.unpack(value) }
.branch( .branch(
Predicate { _, value -> value.containsKey(JSON.record) }, Predicate { _, value -> value.containsKey(JSON.recordTag) },
Predicate { _, value -> value.containsKey(JSON.institution) }, Predicate { _, value -> value.containsKey(JSON.institutionTag) },
Predicate { _, value -> value.containsKey(JSON.recordSet) }, Predicate { _, value -> value.containsKey(JSON.recordSetTag) },
Predicate { _, _ -> true } Predicate { _, _ -> true }
) )
......
...@@ -53,7 +53,7 @@ object Filter { ...@@ -53,7 +53,7 @@ object Filter {
} }
} }
is JsonObject -> is JsonObject ->
when (val newValue = fullGraph[it["@id"]]) { when (val newValue = fullGraph[it[KEYS.entityId]]) {
is JsonObject -> listOf(newValue) is JsonObject -> listOf(newValue)
else -> { else -> {
log.error("Could not extract json-object from entity $it in property $sourceProperty.") log.error("Could not extract json-object from entity $it in property $sourceProperty.")
...@@ -64,16 +64,16 @@ object Filter { ...@@ -64,16 +64,16 @@ object Filter {
it.mapNotNull { child -> it.mapNotNull { child ->
when (child) { when (child) {
is String -> fullGraph[child] is String -> fullGraph[child]
is JsonObject -> fullGraph[child["@id"]] is JsonObject -> fullGraph[child[KEYS.entityId]]
else -> null else -> null
} }
} }
null -> { null -> {
log.info("Property $sourceProperty is not defined in resource ${sourceResource["@id"]}.") log.info("Property $sourceProperty is not defined in resource ${sourceResource[KEYS.entityId]}.")
emptyList() emptyList()
} }
else -> { else -> {
log.error("Unexpected JSON structure for property $sourceProperty in resource ${sourceResource["@id"]}.") log.error("Unexpected JSON structure for property $sourceProperty in resource ${sourceResource[KEYS.entityId]}.")
emptyList() emptyList()
} }
} }
......
...@@ -24,9 +24,9 @@ import org.memobase.model.FacetContainer ...@@ -24,9 +24,9 @@ import org.memobase.model.FacetContainer
import org.memobase.model.LanguageContainer import org.memobase.model.LanguageContainer
/** /**
Extraction helpers for institutions and record sets. * Extraction helpers for institutions and record sets.
*/ */
object Meta { object InstitutionAndRecordSetExtractionHelper {
private val log = LogManager.getLogger("InstitutionAndRecordSetExtractionHelper") private val log = LogManager.getLogger("InstitutionAndRecordSetExtractionHelper")
fun extractInstitution(record: JsonObject): List<FacetContainer> { fun extractInstitution(record: JsonObject): List<FacetContainer> {
......
...@@ -30,16 +30,13 @@ object JSON { ...@@ -30,16 +30,13 @@ object JSON {
private val log = LogManager.getLogger("JsonParser") private val log = LogManager.getLogger("JsonParser")
private const val graph = "@graph" private const val graph = "@graph"
private const val atId = "@id"
const val atType = "@type" // rdf:type
const val type = "type" // rico:type
private const val Record = "Record" private const val Record = "Record"
private const val RecordSet = "RecordSet" private const val RecordSet = "RecordSet"
private const val CorporateBody = "CorporateBody" private const val CorporateBody = "CorporateBody"
private const val memobaseInstitution = "memobaseInstitution" private const val memobaseInstitutionType = "memobaseInstitution"
const val institution = "institution" const val institutionTag = "institution"
const val record = "record" const val recordTag = "record"
const val recordSet = "recordSet" const val recordSetTag = "recordSet"
private val klaxon = Klaxon() private val klaxon = Klaxon()
...@@ -55,14 +52,14 @@ object JSON { ...@@ -55,14 +52,14 @@ object JSON {
fun unpack(input: JsonObject): Map<String, JsonObject> { fun unpack(input: JsonObject): Map<String, JsonObject> {
val graph = input[graph] as JsonArray<JsonObject> val graph = input[graph] as JsonArray<JsonObject>
return graph.map { return graph.map {
if (it[atType] == NS.rico + Record) { if (it[KEYS.atType] == NS.rico + Record) {
Pair(record, it) Pair(recordTag, it)
} else if (it[atType] == NS.rico + RecordSet) { } else if (it[KEYS.atType] == NS.rico + RecordSet) {
Pair(recordSet, it) Pair(recordSetTag, it)
} else if (it[atType] == NS.rico + CorporateBody && it[type] == memobaseInstitution) { } else if (it[KEYS.atType] == NS.rico + CorporateBody && it[KEYS.ricoType] == memobaseInstitutionType) {
Pair(institution, it) Pair(institutionTag, it)
} else { } else {
Pair(it[atId] as String, it) Pair(it[KEYS.entityId] as String, it)
} }
}.toMap() }.toMap()
} }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment