Commit ffad8b46 authored by Günter Hipler's avatar Günter Hipler
Browse files

(probably) last changes before productive export to Europena

- use productive backlinks (*.memobase.ch)
- changes in coordinates (from lat/long to wikidata IDs)
-> depending on this change Helpers to handle this
- new configuration files for coorinates (Silvia)
parent 51db06fd
Pipeline #25860 passed with stages
in 7 minutes and 33 seconds
...@@ -17,7 +17,7 @@ kafkacat -C -b mb-ka1:9092 -t fedora-output-json-records -K '\t' -o beginning | ...@@ -17,7 +17,7 @@ kafkacat -C -b mb-ka1:9092 -t fedora-output-json-records -K '\t' -o beginning |
kafkacat -L -b dd-ka1:9092 -t fedora-output-json-records -K '\t' -o beginning | gzip > fedora-output1.json.gz kafkacat -L -b dd-ka1:9092 -t fedora-output-json-records -K '\t' -o beginning | gzip > fedora-output1.json.gz
Import des topic Import des topic
docker run --rm -v /home/swissbib/environment/code/repositories/memoriav/gitlab/services/postprocessing/rico-edm-transformer/data:/data -it --network host edenhill/kafkacat:1.6.0 -P -b VPN:9092 -t fedora-output-json-records1 -K '\t' -l /data/fedora-output-20210409.json docker run --rm -v /home/swissbib/environment/code/repositories/memoriav/gitlab/services/postprocessing/rico-edm-transformer/data:/data -it --network host edenhill/kafkacat:1.6.0 -P -b VPN:9092 -t mb-di-processed-records-prod -K '\t' -l /data/mb-di-processed-records-prod.json
......
...@@ -22,6 +22,7 @@ package ch.memobase.rico2edm.edm.subjects ...@@ -22,6 +22,7 @@ package ch.memobase.rico2edm.edm.subjects
import ch.memobase.rico2edm.edm.EDMRdf4jModel import ch.memobase.rico2edm.edm.EDMRdf4jModel
import ch.memobase.rico2edm.rdf.vocabularies.{EDMVocab, OreVocab} import ch.memobase.rico2edm.rdf.vocabularies.{EDMVocab, OreVocab}
import ch.memobase.rico2edm.utils.Keys
import org.eclipse.rdf4j.model.Model import org.eclipse.rdf4j.model.Model
import org.eclipse.rdf4j.model.impl.SimpleValueFactory import org.eclipse.rdf4j.model.impl.SimpleValueFactory
import org.eclipse.rdf4j.model.util.Values.iri import org.eclipse.rdf4j.model.util.Values.iri
...@@ -95,9 +96,9 @@ class ProvidedCHO (val id: String) { ...@@ -95,9 +96,9 @@ class ProvidedCHO (val id: String) {
def addEdmType(edmtype:Option[String]): Unit = def addEdmType(edmtype:Option[String]): Unit =
edmtype.map(t => model.add(iri(id),EDMVocab.TYPE,factory.createLiteral(t))) edmtype.map(t => model.add(iri(id),EDMVocab.TYPE,factory.createLiteral(t)))
def addCurrentLocation(edmCurrentLocation:Option[(String,String)]): Unit = def addCurrentLocation(edmCurrentLocation:Option[String]): Unit =
edmCurrentLocation.map(t => model.add(iri(id),EDMVocab.CURRENT_LOCATION, edmCurrentLocation.map(t => model.add(iri(id),EDMVocab.CURRENT_LOCATION,
factory.createLiteral(s"${t._1} / ${t._2}"))) factory.createIRI(s"${t}")))
def addIsPartOf(dcTermsPartOf:Option[String]): Unit = def addIsPartOf(dcTermsPartOf:Option[String]): Unit =
dcTermsPartOf.map(t => model.add(iri(id),DCTERMS.IS_PART_OF,factory.createLiteral(t))) dcTermsPartOf.map(t => model.add(iri(id),DCTERMS.IS_PART_OF,factory.createLiteral(t)))
...@@ -137,8 +138,7 @@ object WebResource { ...@@ -137,8 +138,7 @@ object WebResource {
if (identValue.matches(shortID)) { if (identValue.matches(shortID)) {
new WebResource(shortID) new WebResource(shortID)
} else { } else {
//new WebResource(s"https://media.memobase.ch/memo/${shortID}/master") new WebResource(s"${Keys.MEMOBASE_MEDIA_SERVICE_WEB_ENPOINT}/memo/${shortID}/master")
new WebResource(s"https://media.memobase.k8s.unibas.ch/memo/${shortID}/master")
} }
...@@ -166,9 +166,7 @@ class Aggregation(private val id: String) { ...@@ -166,9 +166,7 @@ class Aggregation(private val id: String) {
model.add(iri(id),EDMVocab.IS_SHOWN_AT,factory.createIRI(a)) model.add(iri(id),EDMVocab.IS_SHOWN_AT,factory.createIRI(a))
} else { } else {
model.add(iri(id),EDMVocab.IS_SHOWN_AT,factory.createIRI( model.add(iri(id),EDMVocab.IS_SHOWN_AT,factory.createIRI(
//s"https://memobase.ch/object/${a}")) s"${Keys.MEMOBASE_SERVICE_WEB_ENPOINT}/object/${a}"))
s"https://memobase-customer-live.outermedia.net/object/${a}"))
} }
}) })
} }
...@@ -180,8 +178,7 @@ class Aggregation(private val id: String) { ...@@ -180,8 +178,7 @@ class Aggregation(private val id: String) {
model.add(iri(id),EDMVocab.IS_SHOWN_BY,factory.createIRI(a)) model.add(iri(id),EDMVocab.IS_SHOWN_BY,factory.createIRI(a))
} else { } else {
model.add(iri(id),EDMVocab.IS_SHOWN_BY,factory.createIRI( model.add(iri(id),EDMVocab.IS_SHOWN_BY,factory.createIRI(
//s"https://media.memobase.ch/memo/${a}/master")) s"${Keys.MEMOBASE_MEDIA_SERVICE_WEB_ENPOINT}/memo/${a}/master"))
s"https://media.memobase.k8s.unibas.ch/memo/${a}/master"))
} }
}) })
} }
...@@ -192,7 +189,7 @@ class Aggregation(private val id: String) { ...@@ -192,7 +189,7 @@ class Aggregation(private val id: String) {
model.add(iri(id),EDMVocab.OBJECT,factory.createIRI(a)) model.add(iri(id),EDMVocab.OBJECT,factory.createIRI(a))
} else { } else {
model.add(iri(id),EDMVocab.OBJECT,factory.createIRI( model.add(iri(id),EDMVocab.OBJECT,factory.createIRI(
s"https://media.memobase.k8s.unibas.ch/memo/${a}/master")) s"${Keys.MEMOBASE_MEDIA_SERVICE_WEB_ENPOINT}/memo/${a}/master"))
} }
}) })
} }
...@@ -217,7 +214,7 @@ class Aggregation(private val id: String) { ...@@ -217,7 +214,7 @@ class Aggregation(private val id: String) {
} else { } else {
model.add(iri(id),EDMVocab.OBJECT,factory.createIRI( model.add(iri(id),EDMVocab.OBJECT,factory.createIRI(
//s"https://media.memobase.ch/memo/${a}-poster")) //s"https://media.memobase.ch/memo/${a}-poster"))
s"https://media.memobase.k8s.unibas.ch/memo/${a}-poster")) s"${Keys.MEMOBASE_MEDIA_SERVICE_WEB_ENPOINT}/memo/${a}-poster"))
} }
}) })
} }
...@@ -234,7 +231,7 @@ class Aggregation(private val id: String) { ...@@ -234,7 +231,7 @@ class Aggregation(private val id: String) {
object Aggregation { object Aggregation {
//def apply (shortID: String): Aggregation = new Aggregation(s"https://memobase.ch/object/${shortID}") //def apply (shortID: String): Aggregation = new Aggregation(s"https://memobase.ch/object/${shortID}")
def apply (shortID: String): Aggregation = new Aggregation(s"https://memobase-customer-live.outermedia.net/object/${shortID}") def apply (shortID: String): Aggregation = new Aggregation(s"${Keys.MEMOBASE_SERVICE_WEB_ENPOINT}/object/${shortID}")
} }
......
...@@ -34,7 +34,9 @@ object Helper { ...@@ -34,7 +34,9 @@ object Helper {
private var isoLanguageCodes: Option[JHashMap[String,String]] = None private var isoLanguageCodes: Option[JHashMap[String,String]] = None
private var institutionsCoordinates: Option[JHashMap[String,(String,String)]] = None //old data structure for explicit coordinates - last moment change to wikidata URL
//private var institutionsCoordinates: Option[JHashMap[String,(String,String)]] = None
private var institutionsCoordinates: Option[JHashMap[String,String]] = None
private var exportRules: Option[OaiExportRules] = None private var exportRules: Option[OaiExportRules] = None
...@@ -95,6 +97,7 @@ object Helper { ...@@ -95,6 +97,7 @@ object Helper {
} }
/*
def initInstitutionsCoordinateMapping(props: Properties):Unit = { def initInstitutionsCoordinateMapping(props: Properties):Unit = {
val coord = new JHashMap[String,(String,String)]() val coord = new JHashMap[String,(String,String)]()
...@@ -108,6 +111,21 @@ object Helper { ...@@ -108,6 +111,21 @@ object Helper {
institutionsCoordinates = Some(coord) institutionsCoordinates = Some(coord)
} }
*/
def initInstitutionsCoordinateMapping(props: Properties):Unit = {
val coord = new JHashMap[String,String]()
using(io.Source.fromFile(props.get(Keys.INSTITUTIONS_COORDINATES_MAPPING).toString)) { source =>
for (line <- source.getLines) {
val temp = line.split(",").map(_.trim)
coord.put(temp(0),temp(1))
}
}
institutionsCoordinates = Some(coord)
}
//noinspection ScalaStyle //noinspection ScalaStyle
private def using[A <: { def close(): Unit }, B](resource: A)(f: A => B): B = private def using[A <: { def close(): Unit }, B](resource: A)(f: A => B): B =
...@@ -136,7 +154,7 @@ object Helper { ...@@ -136,7 +154,7 @@ object Helper {
list => list.exists { item => {if (exportRules.isDefined) exportRules.get.institutions.contains(item) else false} } list => list.exists { item => {if (exportRules.isDefined) exportRules.get.institutions.contains(item) else false} }
def getInstitutionCoord(institutionId:String):Option[(String,String)] = { def getInstitutionCoord(institutionId:String):Option[String] = {
institutionsCoordinates match { institutionsCoordinates match {
case Some(coords) if coords.containsKey(institutionId) => case Some(coords) if coords.containsKey(institutionId) =>
Some(coords.get(institutionId)) Some(coords.get(institutionId))
......
...@@ -35,4 +35,9 @@ object Keys { ...@@ -35,4 +35,9 @@ object Keys {
val VIMEO_KEY = "VIMEO" val VIMEO_KEY = "VIMEO"
val MEMOBASE_SERVICE_WEB_ENPOINT = "https://memobase.ch"
val MEMOBASE_MEDIA_SERVICE_WEB_ENPOINT = "https://media.memobase.ch"
} }
"aag","47.38787196, 8.049455603" aag,http://www.wikidata.org/entity/Q1410236
"abe","46.95089, 7.43706" abe,http://www.wikidata.org/entity/Q2324779
"acj","47.41728, 7.07422" acj,http://www.wikidata.org/entity/Q14847939
"adg","46.8542, 9.5382" adg,http://www.wikidata.org/entity/Q27479980
"afz","47.375219, 8.545961" afz,http://www.wikidata.org/entity/Q27490201
"agl","47.04153, 9.06705" agl,http://www.wikidata.org/entity/Q14847933
"apf","46.5199, 6.6332" apf,http://www.wikidata.org/entity/Q16529040
"atd","46.186235, 8.732027" atd,http://www.wikidata.org/entity/Q15128093
"ati","46.192846, 9.0132" ati,http://www.wikidata.org/entity/Q14847844
"avl","46.52511, 6.62441" avl,http://www.wikidata.org/entity/Q2860447
"azh","47.39077, 8.512219" azh,http://www.wikidata.org/entity/Q98352593
"baa","45.86725, 8.98326" baa,http://www.wikidata.org/entity/Q27485106
"bab","47.551869, 7.589958" bab,http://www.wikidata.org/entity/Q23783862
"bar","46.940555555, 7.446388888" bar,http://www.wikidata.org/entity/Q679141
"baz","47.37237, 8.54582" baz,http://www.wikidata.org/entity/Q27490205
"bbb","46.9473, 7.4483" bbb,http://www.wikidata.org/entity/Q669509
"bcf","46.8066, 7.15597" bcf,http://www.wikidata.org/entity/Q1728085
"bfl","46.92833333, 7.45166667" bfl,http://www.wikidata.org/entity/Q685592
"bmf","47.5567059, 7.578635" bmf,http://www.wikidata.org/entity/Q98369286
"bpu","46.9902, 6.9311" bpu,http://www.wikidata.org/entity/Q294149
"bvc","47.1042, 6.82634" bvc,http://www.wikidata.org/entity/Q2901561
"cde","46.19309, 9.01245" cde,http://www.wikidata.org/entity/Q3664826
"cdt","46.19908, 6.13594" cdt,http://www.wikidata.org/entity/Q85722326
"cag","46.1992, 6.1376" cag,http://www.wikidata.org/entity/Q683970
"cic","46.2274, 6.1373" cic,http://www.wikidata.org/entity/Q5987345
"clg","46.87994, 8.64185" clg,http://www.wikidata.org/entity/Q98370442
"clu","47.10016, 6.8239" clu,http://www.wikidata.org/entity/Q19952147
"csa","46.60208, 6.53612" csa,http://www.wikidata.org/entity/Q2945048
"fad","46.44019, 8.93677" fad,http://www.wikidata.org/entity/Q56876763
"fer","46.1314399, 8.802462" fer,http://www.wikidata.org/entity/Q98368716
"fgr","46.85014, 9.53323" fgr,http://www.wikidata.org/entity/Q98379995
"fpc","46.194709, 9.024187" fpc,http://www.wikidata.org/entity/Q27479761
"fss","47.4958, 8.7383" fss,http://www.wikidata.org/entity/Q18643284
"gvs","46.88057, 8.64469" gvs,http://www.wikidata.org/entity/Q1518565
"hgk","47.53307, 7.61098" hgk,http://www.wikidata.org/entity/Q98380085
"hsl","47.07143, 8.27772" hsl,http://www.wikidata.org/entity/Q98355761
"ias","46.52378, 6.58423" ias,http://www.wikidata.org/entity/Q98354443
"ikg","46.85111, 9.533846" ikg,http://www.wikidata.org/entity/Q1664903
"ikr","46.22169, 6.12565" ikr,http://www.wikidata.org/entity/Q1069175
"kak","47.3941882, 8.0587246" kak,http://www.wikidata.org/entity/Q62027737
"kbg","46.849522222, 9.533855555" kbg,http://www.wikidata.org/entity/Q1728089
"kek","47.12683, 8.75293" kek,http://www.wikidata.org/entity/Q27480181
"khz","47.370278, 8.548056" khz,http://www.wikidata.org/entity/Q685038
"klu","47.04138, 8.310911" klu,http://www.wikidata.org/entity/Q27485024
"kmm","47.06376, 7.09299" kmm,http://www.wikidata.org/entity/Q27482247
"lfg","47.11458, 8.38545" lfg,http://www.wikidata.org/entity/Q98370803
"lkb","46.94049, 7.44218" lkb,http://www.wikidata.org/entity/Q681911
"lmz","47.379166666, 8.539722222" lmz,http://www.wikidata.org/entity/Q691896
"maa","47.3944147, 8.0452584" maa,http://www.wikidata.org/entity/Q27478171
"mav","46.938154, 7.394621" mav,http://www.wikidata.org/entity/Q1365781
"mcl","46.0043, 8.95327" mcl,http://www.wikidata.org/entity/Q3329511
"mdl","46.38, 6.24018" mdl,http://www.wikidata.org/entity/Q644296
"meg","46.197797222, 6.137313888" meg,http://www.wikidata.org/entity/Q1955752
"mel","46.509824, 6.632767" mel,http://www.wikidata.org/entity/Q675261
"mem","46.1736, 8.81054" mem,http://www.wikidata.org/entity/Q27485157
"mfk","46.94186, 7.45004" mfk,http://www.wikidata.org/entity/Q1954633
"mgb","46.6167, 7.058596" mgb,http://www.wikidata.org/entity/Q3330578
"mgz","47.39077, 8.512219" mgz,http://www.wikidata.org/entity/Q668213
"mhl","46.5221, 6.63491" mhl,http://www.wikidata.org/entity/Q3330589
"mov","46.94134, 7.436" mov,http://www.wikidata.org/entity/Q98371215
"mws","46.23168, 7.35853" mws,http://www.wikidata.org/entity/Q961468
"raf","47.50045, 8.72527" raf,http://www.wikidata.org/entity/Q2126295
"rkk","47.37968, 8.52745" rkk,http://www.wikidata.org/entity/Q98356453
"rra","47.37845, 8.52948" rra,http://www.wikidata.org/entity/Q2126159
"rti","46.00373, 8.9512" rti,http://www.wikidata.org/entity/Q114794
"rtr","46.8520381, 9.5344336" rtr,http://www.wikidata.org/entity/Q688243
"rts","46.2, 6.083333333" rts,http://www.wikidata.org/entity/Q3416973
"rxb","47.53496, 7.59477" rxb,http://www.wikidata.org/entity/Q1025578
"sap","46.949135, 7.436426111" sap,http://www.wikidata.org/entity/Q50920401
"sbb","47.47540096, 8.205857926" sbb,http://www.wikidata.org/entity/Q155336
"sik","47.362485, 8.555264" sik,http://www.wikidata.org/entity/Q2256716
"snb","46.941444, 7.449667" snb,http://www.wikidata.org/entity/Q201787
"snp","46.00605, 8.9399" snp,http://www.wikidata.org/entity/Q2256277
"son","46.94814, 7.45241" son,http://www.wikidata.org/entity/Q98380014
"soz","47.366827777, 8.547530555" soz,http://www.wikidata.org/entity/Q2256734
"srf","47.40146, 8.53547" srf,http://www.wikidata.org/entity/Q27490292
"sts","47.6973, 8.6337185" sts,http://www.wikidata.org/entity/Q27488773
"swi","46.943487611, 7.473449888" swi,http://www.wikidata.org/entity/Q568173
"ubb","47.5594, 7.5812" ubb,http://www.wikidata.org/entity/Q683842
"vks","47.052777777, 8.335833333" vks,http://www.wikidata.org/entity/Q670595
"zbz","47.374166666, 8.545277777" zbz,http://www.wikidata.org/entity/Q190260
"zem","46.96684, 7.45468" zem,http://www.wikidata.org/entity/Q28500447
\ No newline at end of file \ No newline at end of file
"aag","47.38787196, 8.049455603"
"abe","46.95089, 7.43706"
"acj","47.41728, 7.07422"
"adg","46.8542, 9.5382"
"afz","47.375219, 8.545961"
"agl","47.04153, 9.06705"
"apf","46.5199, 6.6332"
"atd","46.186235, 8.732027"
"ati","46.192846, 9.0132"
"avl","46.52511, 6.62441"
"azh","47.39077, 8.512219"
"baa","45.86725, 8.98326"
"bab","47.551869, 7.589958"
"bar","46.940555555, 7.446388888"
"baz","47.37237, 8.54582"
"bbb","46.9473, 7.4483"
"bcf","46.8066, 7.15597"
"bfl","46.92833333, 7.45166667"
"bmf","47.5567059, 7.578635"
"bpu","46.9902, 6.9311"
"bvc","47.1042, 6.82634"
"cde","46.19309, 9.01245"
"cdt","46.19908, 6.13594"
"cag","46.1992, 6.1376"
"cic","46.2274, 6.1373"
"clg","46.87994, 8.64185"
"clu","47.10016, 6.8239"
"csa","46.60208, 6.53612"
"fad","46.44019, 8.93677"
"fer","46.1314399, 8.802462"
"fgr","46.85014, 9.53323"
"fpc","46.194709, 9.024187"
"fss","47.4958, 8.7383"
"gvs","46.88057, 8.64469"
"hgk","47.53307, 7.61098"
"hsl","47.07143, 8.27772"
"ias","46.52378, 6.58423"
"ikg","46.85111, 9.533846"
"ikr","46.22169, 6.12565"
"kak","47.3941882, 8.0587246"
"kbg","46.849522222, 9.533855555"
"kek","47.12683, 8.75293"
"khz","47.370278, 8.548056"
"klu","47.04138, 8.310911"
"kmm","47.06376, 7.09299"
"lfg","47.11458, 8.38545"
"lkb","46.94049, 7.44218"
"lmz","47.379166666, 8.539722222"
"maa","47.3944147, 8.0452584"
"mav","46.938154, 7.394621"
"mcl","46.0043, 8.95327"
"mdl","46.38, 6.24018"
"meg","46.197797222, 6.137313888"
"mel","46.509824, 6.632767"
"mem","46.1736, 8.81054"
"mfk","46.94186, 7.45004"
"mgb","46.6167, 7.058596"
"mgz","47.39077, 8.512219"
"mhl","46.5221, 6.63491"
"mov","46.94134, 7.436"
"mws","46.23168, 7.35853"
"raf","47.50045, 8.72527"
"rkk","47.37968, 8.52745"
"rra","47.37845, 8.52948"
"rti","46.00373, 8.9512"
"rtr","46.8520381, 9.5344336"
"rts","46.2, 6.083333333"
"rxb","47.53496, 7.59477"
"sap","46.949135, 7.436426111"
"sbb","47.47540096, 8.205857926"
"sik","47.362485, 8.555264"
"snb","46.941444, 7.449667"
"snp","46.00605, 8.9399"
"son","46.94814, 7.45241"
"soz","47.366827777, 8.547530555"
"srf","47.40146, 8.53547"
"sts","47.6973, 8.6337185"
"swi","46.943487611, 7.473449888"
"ubb","47.5594, 7.5812"
"vks","47.052777777, 8.335833333"
"zbz","47.374166666, 8.545277777"
"zem","46.96684, 7.45468"
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment