In order to mitigate against the brute force attacks against Gitlab accounts, we are moving to all edu-ID Logins. We would like to remind you to link your account with your edu-id. Login will be possible only by edu-ID after November 30, 2021. Here you can find the instructions for linking your account.

If you don't have a SWITCH edu-ID, you can create one with this guide here

kind regards

This Server has been upgraded to GitLab release 14.2.6

Commit c5ffd3d1 authored by Jonas Waeber's avatar Jonas Waeber
Browse files

Refactor InstitutionSearchDoc

parent 3b73e501
......@@ -24,6 +24,7 @@ import org.apache.logging.log4j.LogManager
import org.memobase.helpers.Extract
import org.memobase.helpers.InstitutionTypeMapper
import org.memobase.helpers.KEYS
import org.memobase.model.FacetContainer
import org.memobase.model.InstitutionSearchDoc
import org.memobase.model.LanguageContainer
import org.memobase.model.Schema
......@@ -37,7 +38,7 @@ class InstitutionSearchDocBuilder(path: String) {
val institution =
input["institution"] ?: throw InvalidInputException("No institution entity found in message $key.")
val identifiers = mutableListOf<JsonObject>()
val cantons = mutableListOf<LanguageContainer>()
val cantons = mutableListOf<FacetContainer>()
val municipalities = mutableListOf<LanguageContainer>()
val addresses = mutableListOf<String>()
val postalCodes = mutableListOf<String>()
......@@ -45,7 +46,7 @@ class InstitutionSearchDocBuilder(path: String) {
input.values.forEach {
when {
it[KEYS.ricoType] == KEYS.LocationType.canton -> {
cantons.add(extractAdminUnit(it, "canton", "NoCantonFound"))
cantons.add(FacetContainer(extractAdminUnit(it, "canton", "NoCantonFound"), null, emptyList()))
}
it[KEYS.ricoType] == KEYS.LocationType.municipality -> {
municipalities.add(extractAdminUnit(it, "municipality", "NoCityFound"))
......@@ -73,9 +74,9 @@ class InstitutionSearchDocBuilder(path: String) {
val name = extractLanguageContainer(institution[KEYS.name], "NoNameFound")
val description = extractLanguageContainer(institution[KEYS.descriptiveNote], "NoDescriptionFound")
val id = Extract.extractIdValue(identifiers, KEYS.IdentifierType.main) ?: "NoIdentifierFound"
return InstitutionSearchDoc(
institutionId = Extract.extractIdValue(identifiers, KEYS.IdentifierType.main) ?: "NoIdentifierFound",
institutionId = id,
published = institution[KEYS.isPublished].let {
when (it) {
is String -> it.toBoolean()
......@@ -88,14 +89,30 @@ class InstitutionSearchDocBuilder(path: String) {
type = type,
name = name,
description = description,
documentType = listOf(LanguageContainer.placeholder("PLACEHOLDER")),
documentType = listOf(FacetContainer.placeholder("PLACEHOLDER")),
keyVisualLink = institution[KEYS.wikidataImage].let { if (it != null) it as String else "NoKeyVisualLinkDefined" },
canton = cantons,
city = municipalities,
address = addresses,
postalCodes = postalCodes,
numberOfRecordSets = Extract.listOfStrings(institution[KEYS.isHolderOf]).count(),
numberOfDocuments = 0
numberOfDocuments = 0,
teaserColor = institution[KEYS.teaserColor].let {
if (it == null) {
log.error("No teaser colour found for institution $id.")
"#ffffff"
}
else
it as String
},
teaserColorComputed = institution[KEYS.teaserColorComputed].let {
if (it == null) {
log.error("No computed teaser colour found for institution $id.")
"#ffffff"
}
else
it as String
}
)
}
......
......@@ -21,7 +21,7 @@ package org.memobase.builders
import com.beust.klaxon.JsonObject
import org.memobase.helpers.KEYS
import org.memobase.helpers.Extract
import org.memobase.model.FacettedContainer
import org.memobase.model.FacetContainer
import org.memobase.rdf.NS
/**
......@@ -34,7 +34,7 @@ class FacettedContainerBuilder(
private val facetFunction: (jsonObject: JsonObject) -> Pair<String?, List<String>>
) : IFieldBuilder {
private val containers = mutableListOf<FacettedContainer>()
private val containers = mutableListOf<FacetContainer>()
override fun filter(jsonObject: JsonObject): Boolean {
if (jsonObject[KEYS.atType] != NS.rico + targetType) return false
......@@ -44,7 +44,7 @@ class FacettedContainerBuilder(
override fun append(key: String, jsonObject: JsonObject): String {
val filterAndFacet = facetFunction(jsonObject)
containers.add(
FacettedContainer(
FacetContainer(
Extract.languageContainer("", jsonObject[nameProperty])
.reduce { acc, languageContainer -> acc.merge(languageContainer) },
filter = filterAndFacet.first,
......@@ -54,7 +54,7 @@ class FacettedContainerBuilder(
return ""
}
override fun build(): List<FacettedContainer> {
override fun build(): List<FacetContainer> {
return containers
}
}
......@@ -19,17 +19,17 @@
package org.memobase.helpers
import com.beust.klaxon.JsonObject
import org.memobase.model.FacettedContainer
import org.memobase.model.FacetContainer
object CarrierType {
fun extract(items: List<JsonObject>): List<FacettedContainer> {
fun extract(items: List<JsonObject>): List<FacetContainer> {
return items.map {
Extract.listOfStrings(it["sameAs"]).let { value ->
val name = Extract.languageContainer("carrierType", it["name"]).reduce { acc, languageContainer -> acc.merge(languageContainer) }
if (value.isEmpty()) {
FacettedContainer(name, null, emptyList())
FacetContainer(name, null, emptyList())
} else {
FacettedContainer(name, null, value)
FacetContainer(name, null, value)
}
}
}
......
......@@ -17,7 +17,9 @@ object Default {
emptyList(),
"",
0,
0
0,
"",
""
)
val searchDoc = SearchDoc(
......@@ -62,7 +64,7 @@ object Default {
emptyList(),
emptyList(),
emptyList(),
FacettedContainer(LanguageContainer(emptyList(), emptyList(), emptyList(), emptyList()),"", emptyList()),
FacetContainer(LanguageContainer(emptyList(), emptyList(), emptyList(), emptyList()),"", emptyList()),
false,
emptyList(),
emptyList(),
......
......@@ -21,7 +21,7 @@ package org.memobase.helpers
import com.beust.klaxon.JsonArray
import com.beust.klaxon.JsonObject
import org.apache.logging.log4j.LogManager
import org.memobase.model.FacettedContainer
import org.memobase.model.FacetContainer
import org.memobase.model.LanguageContainer
object Extract {
......@@ -90,13 +90,13 @@ object Extract {
}
}
fun facetEntity(entities: List<JsonObject>, property: String): List<FacettedContainer> {
fun facetEntity(entities: List<JsonObject>, property: String): List<FacetContainer> {
return entities.mapNotNull {
val lang = languageContainer(it["@id"] as String, it[property])
if (lang.isEmpty()) {
null
} else {
FacettedContainer(lang.first(), null, lang.first().toList())
FacetContainer(lang.first(), null, lang.first().toList())
}
}
}
......
......@@ -18,13 +18,13 @@
package org.memobase.helpers
import org.memobase.model.LanguageContainer
import org.memobase.model.FacetContainer
class InstitutionTypeMapper(path: String) {
private val labels = LoadFile.readLabelFile(path)
fun getValue(uri: String): LanguageContainer {
fun getValue(uri: String): FacetContainer {
val q = uri.substringAfterLast("/")
return labels[q] ?: LanguageContainer.DEFAULT
return labels[q] ?: FacetContainer.DEFAULT
}
}
......@@ -23,6 +23,9 @@ object KEYS {
const val mediaUrl = "media.url"
const val institutionTypeLabelsPath = "institutionTypeLabelsPath"
}
const val teaserColor = "teaserColor"
const val teaserColorComputed = "teaserColorComputed"
const val entityId = "@id"
const val atType = "@type"
const val isPublished = "isPublished"
......
package org.memobase.helpers
import com.github.doyaaaaaken.kotlincsv.dsl.csvReader
import org.memobase.model.FacetContainer
import org.memobase.model.LanguageContainer
import java.io.File
object LoadFile {
private val csv = csvReader()
fun readLabelFile(path: String): Map<String, LanguageContainer> {
fun readLabelFile(path: String): Map<String, FacetContainer> {
val labelList = csv.readAll(File(path))
val labelsMap = mutableMapOf<String, LanguageContainer>()
val labelsMap = mutableMapOf<String, FacetContainer>()
for (row in labelList.listIterator(1)) {
labelsMap[row[0]] = LanguageContainer(listOf(row[1]), listOf(row[2]), listOf(row[3]), emptyList())
labelsMap[row[0]] = FacetContainer(LanguageContainer(listOf(row[1]), listOf(row[2]), listOf(row[3]), emptyList()), null, emptyList())
}
return labelsMap
}
......
......@@ -2,7 +2,7 @@ package org.memobase.helpers
import com.beust.klaxon.JsonObject
import org.apache.logging.log4j.LogManager
import org.memobase.model.FacettedContainer
import org.memobase.model.FacetContainer
import org.memobase.model.LanguageContainer
import org.memobase.rdf.NS
......@@ -12,16 +12,16 @@ Extraction helpers for institutions and record sets.
object Meta {
private val log = LogManager.getLogger("InstitutionExtraction")
fun extractInstitution(record: JsonObject): List<FacettedContainer> {
fun extractInstitution(record: JsonObject): List<FacetContainer> {
return extract(KEYS.heldBy, record)
}
fun extractRecordSet(record: JsonObject): FacettedContainer {
fun extractRecordSet(record: JsonObject): FacetContainer {
val containers = extract(KEYS.isPartOf, record)
return if (containers.isNotEmpty()) {
containers[0]
} else {
FacettedContainer(
FacetContainer(
LanguageContainer(
listOf("Unknown RecordSet"),
listOf("Unknown RecordSet"),
......@@ -35,18 +35,18 @@ object Meta {
}
// TODO: Implement function to extract labels from elastic index.
private fun extract(key: String, record: JsonObject): List<FacettedContainer> {
private fun extract(key: String, record: JsonObject): List<FacetContainer> {
return record[key].let { items ->
when (items) {
is String -> listOf(
FacettedContainer(
FacetContainer(
LanguageContainer(emptyList(), emptyList(), emptyList(), emptyList()),
null,
listOf(removeNamespace(key, items))
)
)
is List<*> -> items.map { item ->
FacettedContainer(
FacetContainer(
LanguageContainer(emptyList(), emptyList(), emptyList(), emptyList()),
null,
listOf(removeNamespace(key, item as String))
......
......@@ -21,8 +21,24 @@ package org.memobase.model
import com.fasterxml.jackson.annotation.JsonInclude
@JsonInclude(JsonInclude.Include.NON_EMPTY)
data class FacettedContainer(
data class FacetContainer(
val name: LanguageContainer,
val filter: String?,
val facet: List<String>
)
) {
companion object {
val DEFAULT = FacetContainer(
LanguageContainer.DEFAULT,
null,
emptyList()
)
fun placeholder(placeholder: String): FacetContainer {
return FacetContainer(
LanguageContainer.placeholder(placeholder),
placeholder,
listOf(placeholder)
)
}
}
}
......@@ -17,12 +17,14 @@ data class InstitutionSearchDoc(
val postalCodes: List<String>,
// Facettes
val canton: List<LanguageContainer>,
val type: List<LanguageContainer>,
val documentType: List<LanguageContainer>,
val canton: List<FacetContainer>,
val type: List<FacetContainer>,
val documentType: List<FacetContainer>,
// Search Result Display
val keyVisualLink: String,
val numberOfRecordSets: Int,
val numberOfDocuments: Int
val numberOfDocuments: Int,
val teaserColor: String,
val teaserColorComputed: String
) : Schema(institutionId)
\ No newline at end of file
......@@ -42,11 +42,11 @@ data class SearchDoc(
val sameAs: List<String>,
// Concepts
val genre: List<FacettedContainer>,
val keywords: List<FacettedContainer>,
val genre: List<FacetContainer>,
val keywords: List<FacetContainer>,
val languageContent: List<LanguageContainer>,
val languageCaption: List<LanguageContainer>,
val language: List<FacettedContainer>,
val language: List<FacetContainer>,
// Agents
val personSubject: List<AgentWithRelationContainer>,
......@@ -69,8 +69,8 @@ data class SearchDoc(
val agentProducer: List<AgentWithRelationContainer>,
// Places
val placeRelated: List<FacettedContainer>,
val placeCapture: List<FacettedContainer>,
val placeRelated: List<FacetContainer>,
val placeCapture: List<FacetContainer>,
val placeFacet: List<String>,
// Dates
......@@ -78,8 +78,8 @@ data class SearchDoc(
val dateCreated: List<DateContainer>,
val dateIssued: List<DateContainer>,
val institution: List<FacettedContainer>,
val recordSet: FacettedContainer,
val institution: List<FacetContainer>,
val recordSet: FacetContainer,
val memoriavClaim: Boolean,
......@@ -105,7 +105,7 @@ data class SearchDoc(
val colourPhysical: List<LanguageContainer>,
val usageConditionsPhysical: List<LanguageContainer>,
val physicalObjectNote: List<LanguageContainer>,
val format: List<FacettedContainer>,
val format: List<FacetContainer>,
// Combined
val access: List<String>,
......
{"published":false,"name":{"de":["RadioX"],"fr":["Test institution Oana (FR)"],"it":["Test institution Oana (IT)"],"un":[]},"description":{"de":["<p>Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem.</p>"],"fr":["<p>test_description&nbsp;(FR)</p>"],"it":["<p>test_description&nbsp;&nbsp;(IT)</p>"],"un":[]},"city":[{"de":["Münchenstein"],"fr":["Münchenstein"],"it":["Münchenstein"],"un":[]},{"de":[],"fr":[],"it":[],"un":["Bern"]}],"address":["Helvetiapl. 5","Oslo-Strasse 8"],"postalCodes":["3005","4142"],"canton":[{"de":["Bern"],"fr":["Berne"],"it":["Berne"],"un":[]},{"de":["Basel-Stadt"],"fr":["Bâle-Ville"],"it":["Bâle-Ville"],"un":[]}],"type":[{"de":["Behörde"],"fr":["autorité"],"it":["autorità │"],"un":[]},{"de":["Archiv"],"fr":["Archives"],"it":["Archivio │"],"un":[]}],"documentType":[{"de":["PLACEHOLDER"],"fr":["PLACEHOLDER"],"it":["PLACEHOLDER"],"un":["PLACEHOLDER"]}],"keyVisualLink":"https://mb-wf1.memobase.unibas.ch/sites/default/files/styles/teaser/public/2020-10/1.jpg?itok=5ncVBnVQ","numberOfRecordSets":1,"numberOfDocuments":0,"id":"RadioX"}
\ No newline at end of file
{"published":false,"name":{"de":["RadioX"],"fr":["Test institution Oana (FR)"],"it":["Test institution Oana (IT)"],"un":[]},"description":{"de":["<p>Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem.</p>"],"fr":["<p>test_description&nbsp;(FR)</p>"],"it":["<p>test_description&nbsp;&nbsp;(IT)</p>"],"un":[]},"city":[{"de":["Münchenstein"],"fr":["Münchenstein"],"it":["Münchenstein"],"un":[]},{"de":[],"fr":[],"it":[],"un":["Bern"]}],"address":["Helvetiapl. 5","Oslo-Strasse 8"],"postalCodes":["3005","4142"],"canton":[{"name":{"de":["Bern"],"fr":["Berne"],"it":["Berne"],"un":[]}},{"name":{"de":["Basel-Stadt"],"fr":["Bâle-Ville"],"it":["Bâle-Ville"],"un":[]}}],"type":[{"name":{"de":["Behörde"],"fr":["autorité"],"it":["autorità │"],"un":[]}},{"name":{"de":["Archiv"],"fr":["Archives"],"it":["Archivio │"],"un":[]}}],"documentType":[{"name":{"de":["PLACEHOLDER"],"fr":["PLACEHOLDER"],"it":["PLACEHOLDER"],"un":["PLACEHOLDER"]},"filter":"PLACEHOLDER","facet":["PLACEHOLDER"]}],"keyVisualLink":"https://mb-wf1.memobase.unibas.ch/sites/default/files/styles/teaser/public/2020-10/1.jpg?itok=5ncVBnVQ","numberOfRecordSets":1,"numberOfDocuments":0,"teaserColor":"#ffffff","teaserColorComputed":"#ffffff","id":"RadioX"}
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment