Commit 076e09f3 authored by Jonas Waeber's avatar Jonas Waeber
Browse files

Generate locator id for media server.

parent 2c76df90
Pipeline #15989 passed with stages
in 5 minutes and 18 seconds
......@@ -5,6 +5,7 @@ metadata:
namespace: memobase
data:
APPLICATION_ID: "{{ .Values.deploymentName }}-app"
MEDIA_SERVER_URL: "{{ .Values.mediaServerUrl }}"
TOPIC_IN: "{{ .Values.inputTopic }}"
TOPIC_OUT: "{{ .Values.outputTopic }}"
TOPIC_PROCESS: "{{ .Values.deploymentName }}-reporting"
\ No newline at end of file
......@@ -6,4 +6,6 @@ tag: "latest"
deploymentName: search-doc-service
kafkaConfigs: prod-kafka-bootstrap-servers
outputTopic: search-doc-output-documents
inputTopic: search-doc-input-documents
\ No newline at end of file
inputTopic: search-doc-input-documents
mediaServerUrl: https://media.memobase.k8s.unibas.ch/memo/
\ No newline at end of file
......@@ -19,6 +19,7 @@
package org.memobase
object KEYS {
const val mediaUrlPropName = "media.url"
const val entityId = "@id"
const val atType = "@type"
......@@ -68,4 +69,10 @@ object KEYS {
const val series = "series"
const val broadcast = "broadcast"
}
object IdentifierType {
const val main = "main"
const val oldMemobase = "oldMemobase"
const val original = "original"
const val callNumber = "callNumber"
}
}
......@@ -36,7 +36,7 @@ import org.memobase.settings.SettingsLoader
class KafkaTopology(private val settings: SettingsLoader) {
private val log = LogManager.getLogger("StreamsProcessing")
private val searchDocTransform = SearchDocTransform()
private val searchDocTransform = SearchDocTransform(settings.appSettings.getProperty(KEYS.mediaUrlPropName))
fun build(): Topology {
val builder = StreamsBuilder()
val stream = builder.stream<String, String>(settings.inputTopic)
......
......@@ -25,7 +25,7 @@ import org.memobase.helpers.*
import org.memobase.model.SearchDoc
import org.memobase.rdf.NS
class SearchDocTransform {
class SearchDocTransform(private val mediaUrl: String) {
private val log = LogManager.getLogger("SearchDocTransform")
fun transform(input: Map<String, JsonObject>): SearchDoc {
......@@ -74,7 +74,10 @@ class SearchDocTransform {
val suggestContainerBuilder = SuggestContainerBuilder(hasSubjectIds)
for (item in input.values) {
val digitalIdentifierReferences = Extract.identifiers(digitalObject?.get("identifiedBy"))
val digitalIdentifierEntities = mutableListOf<JsonObject>()
for (item in input.entries) {
for (builder: IFieldBuilder in listOf(
personFacetBuilder,
subjectPersonBuilder,
......@@ -101,9 +104,12 @@ class SearchDocTransform {
temporalBuilder,
suggestContainerBuilder
)) {
if (builder.filter(item)) {
builder.append(item)
if (builder.filter(item.value)) {
builder.append(item.value)
}
if (digitalIdentifierReferences.contains(item.key))
digitalIdentifierEntities.add(item.value)
}
}
......@@ -131,13 +137,24 @@ class SearchDocTransform {
log.error("Found no carrier types for record $id.")
}
val locator = try {
val value = Extract.extractIdValue(digitalIdentifierEntities, KEYS.IdentifierType.main)
if (value == null)
""
else
"${mediaUrl}${value}"
} catch (ex: NoSuchElementException) {
""
}
return SearchDoc(
title = Extract.typedEntityByType(recordTitles, "type", "main", "title"),
seriesTitle = Extract.typedEntityByType(recordTitles, "type", "series", "title"),
broadcastTitle = Extract.typedEntityByType(recordTitles, "type", "broadcast", "title"),
type = record["type"] as String,
sourceID = try {
Extract.extractSourceId(recordIdentifiers)
Extract.extractIdValue(recordIdentifiers, KEYS.IdentifierType.original) ?: "NoSourceIdFound"
} catch (ex: NoSuchElementException) {
log.error("No source id found for record $id")
"NoSourceIdFound"
......@@ -192,7 +209,7 @@ class SearchDocTransform {
durationDigital = Extract.listOfStrings(digitalObject?.get("duration")),
colourDigital = Extract.listOfStrings(digitalObject?.get("P60558")),
digitalObjectNote = Extract.languageContainer("descriptiveNote", digitalObject?.get("descriptiveNote")),
locator = Extract.listOfStrings(digitalObject?.get("locator")),
locator = locator,
usageConditionsDigital = Extract.languageContainer(
"conditionsOfUse",
digitalObject?.get("conditionsOfUse")
......
......@@ -27,7 +27,9 @@ class Service(file: String = "app.yml") {
private val log = LogManager.getLogger("SearchDocService")
val settings = SettingsLoader(
listOf(),
listOf(
KEYS.mediaUrlPropName
),
file,
useStreamsConfig = true
)
......
......@@ -25,7 +25,7 @@ import org.memobase.model.FacettedContainer
import org.memobase.model.LanguageContainer
object Extract {
private val log = LogManager.getLogger("ExtractSearchDoc")
private val log = LogManager.getLogger("ExtractHelper")
private fun languageTag(entity: JsonObject, language: String): List<String> {
return entity["@value"].let {
......@@ -101,10 +101,10 @@ object Extract {
}
}
fun extractSourceId(entities: List<JsonObject>): String {
fun extractIdValue(entities: List<JsonObject>, type: String): String? {
return entities.first {
it["type"] as String == "original"
}["identifier"] as String? ?: error("Identifier contains no property rico:identifier: $entities.")
it["type"] as String == type
}.getOrDefault("identifier", null) as String?
}
fun listOfStrings(value: Any?): List<String> {
......@@ -113,8 +113,11 @@ object Extract {
is List<*> ->
if (value[0] is String)
value as List<String>
else
else {
log.error("Could not extract strings from $value.")
emptyList()
}
else ->
emptyList()
}
......
......@@ -84,7 +84,7 @@ data class SearchDoc(
// Digital Object
val durationDigital: List<String>,
val colourDigital: List<String>,
val locator: List<String>,
val locator: String,
val accessDigital: List<String>,
val usageDigital: List<String>,
val usageDigitalGroup: List<String>,
......
app:
media:
url: ${MEDIA_SERVER_URL?:system}
kafka:
streams:
bootstrap.servers: ${KAFKA_BOOTSTRAP_SERVERS:?system}
......
{"title":[{"de":[],"fr":[],"it":[],"un":["Bern: Altstadt, obere; Bundesplatz 4; Bärenplatz (31); Käfiggässchen; -- Fuhrwerk; Automobil; Lastwagen; Transport, Verkehr; Strassenbeleuchtung"]}],"type":"Foto","sourceID":"216133","id":"https://memobase.ch/record/Burgerbib-Krebser-216133","descriptiveNote":[{"de":[],"fr":[],"it":[],"un":["Datierung: Bundesplatz / Büren-Besitzung<br>"]}],"rightsHolder":[{"de":[],"fr":[],"it":[],"un":["Burgerbibliothek Bern"]}],"sameAs":["http://katalog.burgerbib.ch/detail.aspx?ID=216133"],"keywords":[{"name":{"de":[],"fr":[],"it":[],"un":["Transport, Verkehr"]},"facet":["Transport, Verkehr"]}],"personCreator":[{"name":{"de":[],"fr":[],"it":[],"un":["Anonym"]},"relation":{"de":[],"fr":[],"it":[],"un":["author"]},"filter":"Anonym","facet":["0~A~","1~A~Anonym~"]}],"personContributor":[{"name":{"de":[],"fr":[],"it":[],"un":["Familie Krebser"]},"relation":{"de":[],"fr":[],"it":[],"un":["Collector/Sammler"]},"filter":"Familie Krebser","facet":["0~F~","1~F~Familie Krebser~"]}],"personsFacet":["0~A~","1~A~Anonym~","0~F~","1~F~Familie Krebser~"],"placeRelated":[{"name":{"de":[],"fr":[],"it":[],"un":["Käfiggässchen"]},"filter":"Käfiggässchen","facet":["0~K~","1~K~Käfiggässchen~"]}],"placeFacet":["0~K~","1~K~Käfiggässchen~"],"dateCreated":[{"date":"1900/1909","facet":[]}],"institution":[{"name":{"de":[],"fr":[],"it":[],"un":[]},"facet":["Burgerbib"]}],"recordSet":{"name":{"de":[],"fr":[],"it":[],"un":[]},"facet":["Burgerbib-Krebser"]},"memoriavClaim":true,"callNumber":["Historische Sammlung Krebser 13/2"],"physicalCharacteristics":[{"de":[],"fr":[],"it":[],"un":["Weite: 85 mm","Höhe: 100 mm"]}],"format":[{"name":{"de":["Glasplatte"],"fr":[],"it":[],"un":[]},"facet":["http://www.wikidata.org/entity/Q1138868"]},{"name":{"de":[],"fr":[],"it":[],"un":["Glasplatte"]},"facet":[]}],"published":false,"suggest":{"title":["Bern: Altstadt, obere; Bundesplatz 4; Bärenplatz (31); Käfiggässchen; -- Fuhrwerk; Automobil; Lastwagen; Transport, Verkehr; Strassenbeleuchtung"],"seriesTitle":[],"broadcastTitle":[],"keywords":["Transport, Verkehr"]}}
\ No newline at end of file
{"title":[{"de":[],"fr":[],"it":[],"un":["Bern: Altstadt, obere; Bundesplatz 4; Bärenplatz (31); Käfiggässchen; -- Fuhrwerk; Automobil; Lastwagen; Transport, Verkehr; Strassenbeleuchtung"]}],"type":"Foto","sourceID":"216133","id":"https://memobase.ch/record/Burgerbib-Krebser-216133","descriptiveNote":[{"de":[],"fr":[],"it":[],"un":["Datierung: Bundesplatz / Büren-Besitzung<br>"]}],"rightsHolder":[{"de":[],"fr":[],"it":[],"un":["Burgerbibliothek Bern"]}],"sameAs":["http://katalog.burgerbib.ch/detail.aspx?ID=216133"],"keywords":[{"name":{"de":[],"fr":[],"it":[],"un":["Transport, Verkehr"]},"facet":["Transport, Verkehr"]}],"personCreator":[{"name":{"de":[],"fr":[],"it":[],"un":["Anonym"]},"relation":{"de":[],"fr":[],"it":[],"un":["author"]},"filter":"Anonym","facet":["0~A~","1~A~Anonym~"]}],"personContributor":[{"name":{"de":[],"fr":[],"it":[],"un":["Familie Krebser"]},"relation":{"de":[],"fr":[],"it":[],"un":["Collector/Sammler"]},"filter":"Familie Krebser","facet":["0~F~","1~F~Familie Krebser~"]}],"personsFacet":["0~A~","1~A~Anonym~","0~F~","1~F~Familie Krebser~"],"placeRelated":[{"name":{"de":[],"fr":[],"it":[],"un":["Käfiggässchen"]},"filter":"Käfiggässchen","facet":["0~K~","1~K~Käfiggässchen~"]}],"placeFacet":["0~K~","1~K~Käfiggässchen~"],"dateCreated":[{"date":"1900/1909","facet":[]}],"institution":[{"name":{"de":[],"fr":[],"it":[],"un":[]},"facet":["Burgerbib"]}],"recordSet":{"name":{"de":[],"fr":[],"it":[],"un":[]},"facet":["Burgerbib-Krebser"]},"memoriavClaim":true,"locator":"https://media.memobase.k8s.unibas.ch/memo/Burgerbib-Krebser-216133-1","callNumber":["Historische Sammlung Krebser 13/2"],"physicalCharacteristics":[{"de":[],"fr":[],"it":[],"un":["Weite: 85 mm","Höhe: 100 mm"]}],"format":[{"name":{"de":["Glasplatte"],"fr":[],"it":[],"un":[]},"facet":["http://www.wikidata.org/entity/Q1138868"]},{"name":{"de":[],"fr":[],"it":[],"un":["Glasplatte"]},"facet":[]}],"published":false,"suggest":{"title":["Bern: Altstadt, obere; Bundesplatz 4; Bärenplatz (31); Käfiggässchen; -- Fuhrwerk; Automobil; Lastwagen; Transport, Verkehr; Strassenbeleuchtung"],"seriesTitle":[],"broadcastTitle":[],"keywords":["Transport, Verkehr"]}}
\ No newline at end of file
{"title":[{"de":[],"fr":[],"it":[],"un":["Anonym -- Armut"]}],"type":"Foto","sourceID":"208576","id":"https://memobase.ch/record/Burgerbib-Krebser-208576","abstract":[{"de":[],"fr":[],"it":[],"un":["Bildeintrag: auf der Verpackung: 49543<br>"]}],"descriptiveNote":[{"de":[],"fr":[],"it":[],"un":["Datierung: Negativnummer<br>"]}],"rightsHolder":[{"de":[],"fr":[],"it":[],"un":["Burgerbibliothek Bern"]}],"sameAs":["http://katalog.burgerbib.ch/detail.aspx?ID=208576"],"keywords":[{"name":{"de":[],"fr":[],"it":[],"un":["Armut"]},"facet":["Armut"]}],"personCreator":[{"name":{"de":[],"fr":[],"it":[],"un":["Moeglé, Jean"]},"relation":{"de":[],"fr":[],"it":[],"un":["author"]},"filter":"Moeglé, Jean","facet":["0~M~","1~M~Moeglé, Jean~"]}],"personContributor":[{"name":{"de":[],"fr":[],"it":[],"un":["Familie Krebser"]},"relation":{"de":[],"fr":[],"it":[],"un":["Collector/Sammler"]},"filter":"Familie Krebser","facet":["0~F~","1~F~Familie Krebser~"]}],"personsFacet":["0~F~","1~F~Familie Krebser~","0~M~","1~M~Moeglé, Jean~"],"dateCreated":[{"date":"20. Jh.","facet":[]}],"institution":[{"name":{"de":[],"fr":[],"it":[],"un":[]},"facet":["Burgerbib"]}],"recordSet":{"name":{"de":[],"fr":[],"it":[],"un":[]},"facet":["Burgerbib-Krebser"]},"memoriavClaim":true,"callNumber":["Historische Sammlung Krebser 64/21"],"physicalCharacteristics":[{"de":[],"fr":[],"it":[],"un":["Weite: 165 mm","Höhe: 120 mm"]}],"format":[{"name":{"de":["Glasplatte"],"fr":[],"it":[],"un":[]},"facet":["http://www.wikidata.org/entity/Q1138868"]},{"name":{"de":[],"fr":[],"it":[],"un":["Glasplatte"]},"facet":[]}],"published":false,"suggest":{"title":["Anonym -- Armut"],"seriesTitle":[],"broadcastTitle":[],"keywords":["Armut"]}}
\ No newline at end of file
{"title":[{"de":[],"fr":[],"it":[],"un":["Anonym -- Armut"]}],"type":"Foto","sourceID":"208576","id":"https://memobase.ch/record/Burgerbib-Krebser-208576","abstract":[{"de":[],"fr":[],"it":[],"un":["Bildeintrag: auf der Verpackung: 49543<br>"]}],"descriptiveNote":[{"de":[],"fr":[],"it":[],"un":["Datierung: Negativnummer<br>"]}],"rightsHolder":[{"de":[],"fr":[],"it":[],"un":["Burgerbibliothek Bern"]}],"sameAs":["http://katalog.burgerbib.ch/detail.aspx?ID=208576"],"keywords":[{"name":{"de":[],"fr":[],"it":[],"un":["Armut"]},"facet":["Armut"]}],"personCreator":[{"name":{"de":[],"fr":[],"it":[],"un":["Moeglé, Jean"]},"relation":{"de":[],"fr":[],"it":[],"un":["author"]},"filter":"Moeglé, Jean","facet":["0~M~","1~M~Moeglé, Jean~"]}],"personContributor":[{"name":{"de":[],"fr":[],"it":[],"un":["Familie Krebser"]},"relation":{"de":[],"fr":[],"it":[],"un":["Collector/Sammler"]},"filter":"Familie Krebser","facet":["0~F~","1~F~Familie Krebser~"]}],"personsFacet":["0~F~","1~F~Familie Krebser~","0~M~","1~M~Moeglé, Jean~"],"dateCreated":[{"date":"20. Jh.","facet":[]}],"institution":[{"name":{"de":[],"fr":[],"it":[],"un":[]},"facet":["Burgerbib"]}],"recordSet":{"name":{"de":[],"fr":[],"it":[],"un":[]},"facet":["Burgerbib-Krebser"]},"memoriavClaim":true,"locator":"https://media.memobase.k8s.unibas.ch/memo/Burgerbib-Krebser-208576-1","callNumber":["Historische Sammlung Krebser 64/21"],"physicalCharacteristics":[{"de":[],"fr":[],"it":[],"un":["Weite: 165 mm","Höhe: 120 mm"]}],"format":[{"name":{"de":["Glasplatte"],"fr":[],"it":[],"un":[]},"facet":["http://www.wikidata.org/entity/Q1138868"]},{"name":{"de":[],"fr":[],"it":[],"un":["Glasplatte"]},"facet":[]}],"published":false,"suggest":{"title":["Anonym -- Armut"],"seriesTitle":[],"broadcastTitle":[],"keywords":["Armut"]}}
\ No newline at end of file
app:
media:
url: "https://media.memobase.k8s.unibas.ch/memo/"
kafka:
streams:
bootstrap.servers: localhost:12345
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment