Skip to content

GitLab

Commit 27fa2f95 authored by Günter Hipler's avatar Günter Hipler
Browse files

last changes before first prototype is deployed on kafka

parent 027391e9
Pipeline #23095 passed with stages
in 6 minutes and 11 seconds
Hide whitespace changes
Inline Side-by-side
gh/create.index.json.tool.txt
View file @ 27fa2f95
curl -XPUT "http://localhost:8080/oai-v1" -H 'Content-Type: application/json' -d'
curl -XPUT "http://mb-es1.memobase.unibas.ch:8080/oai-v1" -H 'Content-Type: application/json' -d'
{
"settings": {
"number_of_replicas": 1,
......
gh/ergebnisse/20210311.cho.webresource.aggregation.json 0 → 100644
View file @ 27fa2f95
{
"took" : 1,
"timed_out" : false,
"_shards" : {
"total" : 3,
"successful" : 3,
"skipped" : 0,
"failed" : 0
},
"hits" : {
"total" : {
"value" : 1,
"relation" : "eq"
},
"max_score" : 1.0,
"hits" : [
{
"_index" : "oai-v1",
"_type" : "_doc",
"_id" : "rts-002-FNB023_1639_023",
"_score" : 1.0,
"_source" : {
"id" : "https://memobase.ch/record/rts-002-FNB023_1639_023",
"document" : """
<rdf:RDF
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:dcterms="http://purl.org/dc/terms/"
xmlns:edm="http://www.europeana.eu/schemas/edm/"
xmlns:ore="http://www.openarchives.org/ore/terms/"
xmlns:owl="http://www.w3.org/2002/07/owl#"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:foaf="http://xmlns.com/foaf/0.1/"
xmlns:wgs84_pos="http://www.w3.org/2004/02/skos/core#"
xmlns:crm="http://www.cidoc‐crm.org/cidoc‐crm/"
xmlns:cc="http://creativecommons.org/ns#">
<edm:ProvidedCHO rdf:about="https://memobase.ch/record/rts-002-FNB023_1639_023">
<rdf:type rdf:resource="http://www.europeana.eu/schemas/edm/ProvidedCHO"/>
<dc:title>DAT1/01: Appel à la discipline lancé par le président René Coty et Pierre Pflimlin à la suite du coup d'état en Algérie; DAT1/02-DAT1/04: Déclaration de Pierre Pflimlin</dc:title>
<dc:description>Sonorer Hintergrund: Marseillaise &lt;br&gt;</dc:description>
<dc:description>Depot Träger: Lausanne: Phonothèque RSR &lt;br&gt; FONOTECA NAZIONALE SVIZZERA, Lugano; PHONOTHEQUE NATIONALE SUISSE; SCHWEIZERISCHE LANDESPHONOTHEK; SWISS NATIONAL SOUND ARCHIVES &lt;br&gt; Les données proviennent de la version ultérieure de Memobase.</dc:description>
<dc:creator>Author / Studio Radio Lausanne</dc:creator>
<dc:contributor>Orateur / Responsable du Cabinet des Affaires étrangères / inconnu</dc:contributor>
<dc:contributor>Orateur / Président de la République / COTY, René</dc:contributor>
<dc:contributor>Présentateur / Journaliste / inconnu</dc:contributor>
<dc:contributor>Orateur / Président du Gouvernement / PFLIMLIN, Pierre</dc:contributor>
<dc:contributor>Studio Radio Lausanne</dc:contributor>
<dc:identifier>FNB023_1639.023</dc:identifier>
<dc:identifier>rts-002-FNB023_1639_023</dc:identifier>
<dc:language>Français</dc:language>
<dc:language>French</dc:language>
<dc:language>Francese</dc:language>
<dc:language>Französisch</dc:language>
<dc:publisher>[Aucune information]</dc:publisher>
<dc:rights>SOCIETA SVIZZERA DI RADIODIFFUSIONE E TELEVISIONE, Losanna; SOCIETE SUISSE DE RADIODIFFUSION ET TELEVISION, Lausanne; SCHWEIZERISCHE RADIO-UND FERNSEHGESELLSCHAFT; SWISS BROADCASTING CORPORATION; SRG SSR IDEE SUISSE</dc:rights>
<dc:subject>Coup d'état en Algérie</dc:subject>
<dc:subject>BORDENEUVE</dc:subject>
<dc:subject>MOREAU, Jean</dc:subject>
<dc:subject>FAURE</dc:subject>
<dc:subject>Intervista</dc:subject>
<dc:subject>Parlato</dc:subject>
<dc:subject>Rede</dc:subject>
<dc:subject>Gesprochen</dc:subject>
<dc:subject>Interview</dc:subject>
<dc:subject>Parlé</dc:subject>
<dc:subject>Discours</dc:subject>
<dc:subject>Discorso</dc:subject>
<dcterms:created>1958-05-14; 1958-05-12</dcterms:created>
<dcterms:medium>78 T Direktschnittplatte</dcterms:medium>
<dcterms:spatial>Alger</dcterms:spatial>
<dcterms:spatial>France</dcterms:spatial>
<dcterms:spatial>Algérie</dcterms:spatial>
<dcterms:temporal>1958-05-14-1958-05-14</dcterms:temporal>
<edm:type>SOUND</edm:type>
</edm:ProvidedCHO>
<edm:WebResource rdf:about="https://memobase.ch/digital/rts-002-FNB023_1639_023-1">
<rdf:type rdf:resource="http://www.europeana.eu/schemas/edm/WebResource"/>
</edm:WebResource>
<ore:Aggregation rdf:about="https://memobase.ch/record/rts-002-FNB023_1639_023_??">
<rdf:type rdf:resource="http://www.openarchives.org/ore/terms/Aggregation"/>
<wgs84_pos:prefLabel>only test for aggregation</wgs84_pos:prefLabel>
</ore:Aggregation>
</rdf:RDF>""",
"format" : "EDM",
"published" : true,
"recordset" : [
"rts-002"
],
"institution" : [
"rts"
],
"lastUpdatedDate" : "2021-03-11T10:39:11.493791Z"
}
}
]
}
}
\ No newline at end of file
k8s-manifests/test/deployment.yml
View file @ 27fa2f95
......@@ -31,7 +31,7 @@ spec:
memory: "256Mi"
env:
- name: JOB_ID
value: iiif-manifest-creator
value: rico-2-edm-transfromer
- name: KAFKA_BOOTSTRAP_SERVERS
value: mb-ka1.memobase.unibas.ch:9092,mb-ka2.memobase.unibas.ch:9092,mb-ka3.memobase.unibas.ch:9092
- name: APPLICATION_ID
......@@ -39,7 +39,7 @@ spec:
- name: TOPIC_IN
value: fedora-output-json-records
- name: TOPIC_OUT
value: edm-documents
value: edm-es-records
- name: TOPIC_PROCESS
value: postprocessing-reporting
restartPolicy: Always
src/main/scala/ch/memobase/edm/EDM.scala
View file @ 27fa2f95
......@@ -51,16 +51,16 @@ class EDM {
val webExtraction = createWebResources(graph,record.get,digitalObject)
//actually only one instance - correct?
val placeExtraction: ExtractionResult[Place] = createPlace(graph,record.get,digitalObject)
//val placeExtraction: ExtractionResult[Place] = createPlace(graph,record.get,digitalObject)
val aggregationExtraction: ExtractionResult[Aggregation] = createAggregation(graph,record.get,digitalObject)
val conceptExtraction: ExtractionResult[Concept] = createConcept(graph,record.get,digitalObject)
val timespanExtraction: ExtractionResult[TimeSpan] = createTimeSpan(graph,record.get,digitalObject)
//val conceptExtraction: ExtractionResult[Concept] = createConcept(graph,record.get,digitalObject)
//val timespanExtraction: ExtractionResult[TimeSpan] = createTimeSpan(graph,record.get,digitalObject)
webExtraction.obj.foreach(webResource =>
choExtraction.obj.getModel.addAll(webResource.getModel)
)
choExtraction.obj.getModel.addAll(placeExtraction.obj.getModel)
//choExtraction.obj.getModel.addAll(aggregationExtraction.obj.getModel)
//choExtraction.obj.getModel.addAll(placeExtraction.obj.getModel)
choExtraction.obj.getModel.addAll(aggregationExtraction.obj.getModel)
//choExtraction.obj.getModel.addAll(conceptExtraction.obj.getModel)
//choExtraction.obj.getModel.addAll(timespanExtraction.obj.getModel)
......@@ -210,7 +210,8 @@ class EDM {
//how many places instances are available??
//is it correct to use recordId
val aggregation = new Aggregation(Extractors.recordId(record).get)
//use apply method because we do not know how to build the identifier for Aggregations right now
val aggregation = Aggregation(Extractors.recordId(record).get)
//val aggregation = new Aggregation("http://iwerk.ch/4567")
aggregation.addSkosPrefLabel(Some("only test for aggregation"))
......
src/main/scala/ch/memobase/edm/guenterold/Extractors.scala deleted 100644 → 0
View file @ 027391e9
/*
* IIIF Manifest Creator
* Copyright (C) 2020 Memobase
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package ch.memobase.edm.guenterold
import ujson.Value.{Value => JValue}
import ujson.{Arr => JArr, Obj => JObj, Str => JStr}
import scala.collection.mutable
import scala.util.Try
object Extractors {
val jsonGraph: String => Try[JValue] = jsonString =>
Try { ujson.read(jsonString).obj("@graph").arr }
private val fedoraResource =
(jsonGraph: JArr) =>
(resourceType: String) =>
Try {
jsonGraph.value.collectFirst {
case res
if (res.obj
.contains("type") && res.obj("type").str == resourceType) ||
(res.obj.contains("@type") && res
.obj("@type")
.str == resourceType) =>
res.obj.value
}.get
}
private val getBlankNodeContent = (graph: JArr) =>
(resource: mutable.LinkedHashMap[String, JValue]) =>
(property: String) =>
graph.value.collectFirst {
case res
if res.obj("@id") == resource.getOrElse(property, JStr("")) =>
res.obj.value
}
private val getBlankNodesContent = (graph: JArr) =>
(resource: mutable.LinkedHashMap[String, JValue]) =>
(property: String) =>
graph.value
.filter(res =>
res.obj("@id") == resource.getOrElse(property, JStr(""))
)
.map(_.obj.value)
val digitalObject: JArr => Try[mutable.LinkedHashMap[String, JValue]] = {
jsonGraph => fedoraResource(jsonGraph)("digitalObject")
}
val record: JArr => Try[mutable.LinkedHashMap[String, JValue]] = jsonGraph =>
fedoraResource(jsonGraph)(
"https://www.ica.org/standards/RiC/ontology#Record"
)
/*val recordSet: JArr => Try[mutable.LinkedHashMap[String, JValue]] =
jsonGraph => fedoraResource(jsonGraph)("recordSet")*/
private val stringValue = (jsonObj: JObj) =>
(valueKey: String) => jsonObj.value.get(valueKey).flatMap(v => v.strOpt)
private val numValue = (jsonObj: JObj) =>
(valueKey: String) =>
stringValue(jsonObj)(valueKey).flatMap(v => Some(v.toDouble))
private val arrayValues = (jsonObj: JObj) =>
(valueKey: String) =>
jsonObj.value.get(valueKey).flatMap(v => Some(v.arr.toList.map(_.str)))
val ricoType: JObj => Option[String] = record => stringValue(record)("type")
val title: JObj => Option[String] = record => stringValue(record)("title")
val descriptiveNote: JObj => Option[String] = record =>
stringValue(record)("descriptiveNote")
val scopeAndContent: JObj => Option[String] = record =>
stringValue(record)("scopeAndContent")
val publishedBy
: JArr => mutable.LinkedHashMap[String, JValue] => Option[String] =
graph =>
record =>
getBlankNodeContent(graph)(record)("publishedBy").flatMap(v =>
stringValue(v)("name")
)
val placeOfCapture
: JArr => mutable.LinkedHashMap[String, JValue] => Option[String] =
graph =>
record =>
getBlankNodeContent(graph)(record)("P60556").flatMap(v =>
stringValue(v)("name")
)
val spatial: JArr => mutable.LinkedHashMap[String, JValue] => Option[String] =
graph =>
record =>
getBlankNodeContent(graph)(record)("spatial").flatMap(v =>
stringValue(v)("name")
)
val hasLanguage
: JArr => mutable.LinkedHashMap[String, JValue] => Option[String] =
graph =>
record =>
getBlankNodeContent(graph)(record)("hasLanguage").flatMap(v =>
stringValue(v)("name")
)
//noinspection ScalaStyle
val resourceCreator: JArr => mutable.LinkedHashMap[String, JValue] => List[
(String, String)
] =
graph =>
record =>
getBlankNodesContent(graph)(record)(
"recordResourceOrInstantiationIsSourceOfCreationRelation"
)
.flatMap(obj =>
getBlankNodeContent(graph)(obj)("creationRelationHasTarget")
.flatMap(v => stringValue(v)("name")) match {
case Some(name) => Some(obj("name").str, name)
case None => None
}
)
.toList
val producer
: JArr => mutable.LinkedHashMap[String, JValue] => Option[String] =
graph =>
record =>
getBlankNodeContent(graph)(record)("P60441").flatMap(v =>
stringValue(v)("name")
)
val componentColor: JObj => Option[List[String]] = digitalObject =>
arrayValues(digitalObject)("componentColor")
val dctAbstract: JObj => Option[String] = record =>
stringValue(record)("abstract")
val mediaResourceDescription: JObj => Option[String] = digitalObject =>
stringValue(digitalObject)("mediaResourceDescription")
val conditionsOfUse: JObj => Option[String] = record =>
stringValue(record)("conditionsOfUse")
val regulatedByHolder
: JArr => mutable.LinkedHashMap[String, JValue] => Option[String] =
graph =>
digitalObject =>
getBlankNodesContent(graph)(digitalObject)("regulatedBy")
.collectFirst {
case res if res.obj("type").str == "holder" =>
res.obj("name").str
}
val license: JArr => mutable.LinkedHashMap[String, JValue] => Option[String] =
graph =>
digitalObject =>
getBlankNodesContent(graph)(digitalObject)("regulatedBy")
.collectFirst {
case res if res.obj("type").str == "usage" =>
res.obj("name").str
}
/* val logo: JArr => mutable.LinkedHashMap[String, JValue] => Option[String] =
graph =>
recordSet =>
getBlankNodeContent(graph)(recordSet)("heldBy")
.flatMap(v => v.get("P154").flatMap(vi => Some(vi.str))) */
val manifestId: JObj => Option[String] = digitalObject =>
stringValue(digitalObject)("@id").flatMap(v => Some(s"$v/manifest"))
val dobjectId: JObj => Option[String] = digitalObject =>
stringValue(digitalObject)("@id").flatMap(Some(_))
val sequenceId: JObj => Option[String] = digitalObject =>
stringValue(digitalObject)("@id").flatMap(v => Some(s"$v/sequence/default"))
val canvasId: JObj => Option[String] = digitalObject =>
stringValue(digitalObject)("@id").flatMap(v => Some(s"$v/canvas/default"))
val imageResourceId: JObj => Option[String] = digitalObject =>
stringValue(digitalObject)("locator")
val mimeType: JObj => Option[String] = digitalObject =>
stringValue(digitalObject)("hasMimeType")
val imageHeight: JObj => Option[Double] = digitalObject =>
numValue(digitalObject)("height")
val imageWidth: JObj => Option[Double] = digitalObject =>
numValue(digitalObject)("width")
val creationDate
: JArr => mutable.LinkedHashMap[String, JValue] => Option[String] =
graph =>
record =>
getBlankNodeContent(graph)(record)("created").flatMap(v =>
stringValue(v)("normalizedDateValue")
)
val recordId: JObj => Option[String] = record => stringValue(record)("@id").flatMap(Some(_))
}
src/main/scala/ch/memobase/edm/subjects/ProvidedCHO.scala
View file @ 27fa2f95
......@@ -126,6 +126,17 @@ class Aggregation(val id: String) {
}
object Aggregation {
//todo: GH discussion with Silvia 11.3.2021
// we should use the Aggreagatuon subject already in the first prototype
// although at the moment we do not know how to create a unique identifier so
// so it is build a subject of it's own in the EDM document (so to say RDF graph for the
// document
def apply (id: String): Aggregation = new Aggregation(s"${id}_??")
}
class Concept(val id: String) {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment