Skip to content

GitLab

Commit ffad8b46 authored by Günter Hipler's avatar Günter Hipler
Browse files

(probably) last changes before productive export to Europena 

- use productive backlinks (*.memobase.ch)
- changes in coordinates (from lat/long to wikidata IDs)
-> depending on this change Helpers to handle this
- new configuration files for coorinates (Silvia)
parent 51db06fd
Pipeline #25860 passed with stages
in 7 minutes and 33 seconds
Hide whitespace changes
Inline Side-by-side
gh/kafkacatnotes.txt
View file @ ffad8b46
......@@ -17,7 +17,7 @@ kafkacat -C -b mb-ka1:9092 -t fedora-output-json-records -K '\t' -o beginning |
kafkacat -L -b dd-ka1:9092 -t fedora-output-json-records -K '\t' -o beginning | gzip > fedora-output1.json.gz
Import des topic
docker run --rm -v /home/swissbib/environment/code/repositories/memoriav/gitlab/services/postprocessing/rico-edm-transformer/data:/data -it --network host edenhill/kafkacat:1.6.0 -P -b VPN:9092 -t fedora-output-json-records1 -K '\t' -l /data/fedora-output-20210409.json
docker run --rm -v /home/swissbib/environment/code/repositories/memoriav/gitlab/services/postprocessing/rico-edm-transformer/data:/data -it --network host edenhill/kafkacat:1.6.0 -P -b VPN:9092 -t mb-di-processed-records-prod -K '\t' -l /data/mb-di-processed-records-prod.json
......
src/main/scala/ch/memobase/rico2edm/edm/subjects/ProvidedCHO.scala
View file @ ffad8b46
......@@ -22,6 +22,7 @@ package ch.memobase.rico2edm.edm.subjects
import ch.memobase.rico2edm.edm.EDMRdf4jModel
import ch.memobase.rico2edm.rdf.vocabularies.{EDMVocab, OreVocab}
import ch.memobase.rico2edm.utils.Keys
import org.eclipse.rdf4j.model.Model
import org.eclipse.rdf4j.model.impl.SimpleValueFactory
import org.eclipse.rdf4j.model.util.Values.iri
......@@ -95,9 +96,9 @@ class ProvidedCHO (val id: String) {
def addEdmType(edmtype:Option[String]): Unit =
edmtype.map(t => model.add(iri(id),EDMVocab.TYPE,factory.createLiteral(t)))
def addCurrentLocation(edmCurrentLocation:Option[(String,String)]): Unit =
def addCurrentLocation(edmCurrentLocation:Option[String]): Unit =
edmCurrentLocation.map(t => model.add(iri(id),EDMVocab.CURRENT_LOCATION,
factory.createLiteral(s"${t._1} / ${t._2}")))
factory.createIRI(s"${t}")))
def addIsPartOf(dcTermsPartOf:Option[String]): Unit =
dcTermsPartOf.map(t => model.add(iri(id),DCTERMS.IS_PART_OF,factory.createLiteral(t)))
......@@ -137,8 +138,7 @@ object WebResource {
if (identValue.matches(shortID)) {
new WebResource(shortID)
} else {
//new WebResource(s"https://media.memobase.ch/memo/${shortID}/master")
new WebResource(s"https://media.memobase.k8s.unibas.ch/memo/${shortID}/master")
new WebResource(s"${Keys.MEMOBASE_MEDIA_SERVICE_WEB_ENPOINT}/memo/${shortID}/master")
}
......@@ -166,9 +166,7 @@ class Aggregation(private val id: String) {
model.add(iri(id),EDMVocab.IS_SHOWN_AT,factory.createIRI(a))
} else {
model.add(iri(id),EDMVocab.IS_SHOWN_AT,factory.createIRI(
//s"https://memobase.ch/object/${a}"))
s"https://memobase-customer-live.outermedia.net/object/${a}"))
s"${Keys.MEMOBASE_SERVICE_WEB_ENPOINT}/object/${a}"))
}
})
}
......@@ -180,8 +178,7 @@ class Aggregation(private val id: String) {
model.add(iri(id),EDMVocab.IS_SHOWN_BY,factory.createIRI(a))
} else {
model.add(iri(id),EDMVocab.IS_SHOWN_BY,factory.createIRI(
//s"https://media.memobase.ch/memo/${a}/master"))
s"https://media.memobase.k8s.unibas.ch/memo/${a}/master"))
s"${Keys.MEMOBASE_MEDIA_SERVICE_WEB_ENPOINT}/memo/${a}/master"))
}
})
}
......@@ -192,7 +189,7 @@ class Aggregation(private val id: String) {
model.add(iri(id),EDMVocab.OBJECT,factory.createIRI(a))
} else {
model.add(iri(id),EDMVocab.OBJECT,factory.createIRI(
s"https://media.memobase.k8s.unibas.ch/memo/${a}/master"))
s"${Keys.MEMOBASE_MEDIA_SERVICE_WEB_ENPOINT}/memo/${a}/master"))
}
})
}
......@@ -217,7 +214,7 @@ class Aggregation(private val id: String) {
} else {
model.add(iri(id),EDMVocab.OBJECT,factory.createIRI(
//s"https://media.memobase.ch/memo/${a}-poster"))
s"https://media.memobase.k8s.unibas.ch/memo/${a}-poster"))
s"${Keys.MEMOBASE_MEDIA_SERVICE_WEB_ENPOINT}/memo/${a}-poster"))
}
})
}
......@@ -234,7 +231,7 @@ class Aggregation(private val id: String) {
object Aggregation {
//def apply (shortID: String): Aggregation = new Aggregation(s"https://memobase.ch/object/${shortID}")
def apply (shortID: String): Aggregation = new Aggregation(s"https://memobase-customer-live.outermedia.net/object/${shortID}")
def apply (shortID: String): Aggregation = new Aggregation(s"${Keys.MEMOBASE_SERVICE_WEB_ENPOINT}/object/${shortID}")
}
......
src/main/scala/ch/memobase/rico2edm/utils/Helper.scala
View file @ ffad8b46
......@@ -34,7 +34,9 @@ object Helper {
private var isoLanguageCodes: Option[JHashMap[String,String]] = None
private var institutionsCoordinates: Option[JHashMap[String,(String,String)]] = None
//old data structure for explicit coordinates - last moment change to wikidata URL
//private var institutionsCoordinates: Option[JHashMap[String,(String,String)]] = None
private var institutionsCoordinates: Option[JHashMap[String,String]] = None
private var exportRules: Option[OaiExportRules] = None
......@@ -95,6 +97,7 @@ object Helper {
}
/*
def initInstitutionsCoordinateMapping(props: Properties):Unit = {
val coord = new JHashMap[String,(String,String)]()
......@@ -108,6 +111,21 @@ object Helper {
institutionsCoordinates = Some(coord)
}
*/
def initInstitutionsCoordinateMapping(props: Properties):Unit = {
val coord = new JHashMap[String,String]()
using(io.Source.fromFile(props.get(Keys.INSTITUTIONS_COORDINATES_MAPPING).toString)) { source =>
for (line <- source.getLines) {
val temp = line.split(",").map(_.trim)
coord.put(temp(0),temp(1))
}
}
institutionsCoordinates = Some(coord)
}
//noinspection ScalaStyle
private def using[A <: { def close(): Unit }, B](resource: A)(f: A => B): B =
......@@ -136,7 +154,7 @@ object Helper {
list => list.exists { item => {if (exportRules.isDefined) exportRules.get.institutions.contains(item) else false} }
def getInstitutionCoord(institutionId:String):Option[(String,String)] = {
def getInstitutionCoord(institutionId:String):Option[String] = {
institutionsCoordinates match {
case Some(coords) if coords.containsKey(institutionId) =>
Some(coords.get(institutionId))
......
src/main/scala/ch/memobase/rico2edm/utils/Keys.scala
View file @ ffad8b46
......@@ -35,4 +35,9 @@ object Keys {
val VIMEO_KEY = "VIMEO"
val MEMOBASE_SERVICE_WEB_ENPOINT = "https://memobase.ch"
val MEMOBASE_MEDIA_SERVICE_WEB_ENPOINT = "https://media.memobase.ch"
}
src/test/resources/enrichement/coordinates_for_testing.csv
View file @ ffad8b46
"aag","47.38787196, 8.049455603"
"abe","46.95089, 7.43706"
"acj","47.41728, 7.07422"
"adg","46.8542, 9.5382"
"afz","47.375219, 8.545961"
"agl","47.04153, 9.06705"
"apf","46.5199, 6.6332"
"atd","46.186235, 8.732027"
"ati","46.192846, 9.0132"
"avl","46.52511, 6.62441"
"azh","47.39077, 8.512219"
"baa","45.86725, 8.98326"
"bab","47.551869, 7.589958"
"bar","46.940555555, 7.446388888"
"baz","47.37237, 8.54582"
"bbb","46.9473, 7.4483"
"bcf","46.8066, 7.15597"
"bfl","46.92833333, 7.45166667"
"bmf","47.5567059, 7.578635"
"bpu","46.9902, 6.9311"
"bvc","47.1042, 6.82634"
"cde","46.19309, 9.01245"
"cdt","46.19908, 6.13594"
"cag","46.1992, 6.1376"
"cic","46.2274, 6.1373"
"clg","46.87994, 8.64185"
"clu","47.10016, 6.8239"
"csa","46.60208, 6.53612"
"fad","46.44019, 8.93677"
"fer","46.1314399, 8.802462"
"fgr","46.85014, 9.53323"
"fpc","46.194709, 9.024187"
"fss","47.4958, 8.7383"
"gvs","46.88057, 8.64469"
"hgk","47.53307, 7.61098"
"hsl","47.07143, 8.27772"
"ias","46.52378, 6.58423"
"ikg","46.85111, 9.533846"
"ikr","46.22169, 6.12565"
"kak","47.3941882, 8.0587246"
"kbg","46.849522222, 9.533855555"
"kek","47.12683, 8.75293"
"khz","47.370278, 8.548056"
"klu","47.04138, 8.310911"
"kmm","47.06376, 7.09299"
"lfg","47.11458, 8.38545"
"lkb","46.94049, 7.44218"
"lmz","47.379166666, 8.539722222"
"maa","47.3944147, 8.0452584"
"mav","46.938154, 7.394621"
"mcl","46.0043, 8.95327"
"mdl","46.38, 6.24018"
"meg","46.197797222, 6.137313888"
"mel","46.509824, 6.632767"
"mem","46.1736, 8.81054"
"mfk","46.94186, 7.45004"
"mgb","46.6167, 7.058596"
"mgz","47.39077, 8.512219"
"mhl","46.5221, 6.63491"
"mov","46.94134, 7.436"
"mws","46.23168, 7.35853"
"raf","47.50045, 8.72527"
"rkk","47.37968, 8.52745"
"rra","47.37845, 8.52948"
"rti","46.00373, 8.9512"
"rtr","46.8520381, 9.5344336"
"rts","46.2, 6.083333333"
"rxb","47.53496, 7.59477"
"sap","46.949135, 7.436426111"
"sbb","47.47540096, 8.205857926"
"sik","47.362485, 8.555264"
"snb","46.941444, 7.449667"
"snp","46.00605, 8.9399"
"son","46.94814, 7.45241"
"soz","47.366827777, 8.547530555"
"srf","47.40146, 8.53547"
"sts","47.6973, 8.6337185"
"swi","46.943487611, 7.473449888"
"ubb","47.5594, 7.5812"
"vks","47.052777777, 8.335833333"
"zbz","47.374166666, 8.545277777"
"zem","46.96684, 7.45468"
\ No newline at end of file
aag,http://www.wikidata.org/entity/Q1410236
abe,http://www.wikidata.org/entity/Q2324779
acj,http://www.wikidata.org/entity/Q14847939
adg,http://www.wikidata.org/entity/Q27479980
afz,http://www.wikidata.org/entity/Q27490201
agl,http://www.wikidata.org/entity/Q14847933
apf,http://www.wikidata.org/entity/Q16529040
atd,http://www.wikidata.org/entity/Q15128093
ati,http://www.wikidata.org/entity/Q14847844
avl,http://www.wikidata.org/entity/Q2860447
azh,http://www.wikidata.org/entity/Q98352593
baa,http://www.wikidata.org/entity/Q27485106
bab,http://www.wikidata.org/entity/Q23783862
bar,http://www.wikidata.org/entity/Q679141
baz,http://www.wikidata.org/entity/Q27490205
bbb,http://www.wikidata.org/entity/Q669509
bcf,http://www.wikidata.org/entity/Q1728085
bfl,http://www.wikidata.org/entity/Q685592
bmf,http://www.wikidata.org/entity/Q98369286
bpu,http://www.wikidata.org/entity/Q294149
bvc,http://www.wikidata.org/entity/Q2901561
cde,http://www.wikidata.org/entity/Q3664826
cdt,http://www.wikidata.org/entity/Q85722326
cag,http://www.wikidata.org/entity/Q683970
cic,http://www.wikidata.org/entity/Q5987345
clg,http://www.wikidata.org/entity/Q98370442
clu,http://www.wikidata.org/entity/Q19952147
csa,http://www.wikidata.org/entity/Q2945048
fad,http://www.wikidata.org/entity/Q56876763
fer,http://www.wikidata.org/entity/Q98368716
fgr,http://www.wikidata.org/entity/Q98379995
fpc,http://www.wikidata.org/entity/Q27479761
fss,http://www.wikidata.org/entity/Q18643284
gvs,http://www.wikidata.org/entity/Q1518565
hgk,http://www.wikidata.org/entity/Q98380085
hsl,http://www.wikidata.org/entity/Q98355761
ias,http://www.wikidata.org/entity/Q98354443
ikg,http://www.wikidata.org/entity/Q1664903
ikr,http://www.wikidata.org/entity/Q1069175
kak,http://www.wikidata.org/entity/Q62027737
kbg,http://www.wikidata.org/entity/Q1728089
kek,http://www.wikidata.org/entity/Q27480181
khz,http://www.wikidata.org/entity/Q685038
klu,http://www.wikidata.org/entity/Q27485024
kmm,http://www.wikidata.org/entity/Q27482247
lfg,http://www.wikidata.org/entity/Q98370803
lkb,http://www.wikidata.org/entity/Q681911
lmz,http://www.wikidata.org/entity/Q691896
maa,http://www.wikidata.org/entity/Q27478171
mav,http://www.wikidata.org/entity/Q1365781
mcl,http://www.wikidata.org/entity/Q3329511
mdl,http://www.wikidata.org/entity/Q644296
meg,http://www.wikidata.org/entity/Q1955752
mel,http://www.wikidata.org/entity/Q675261
mem,http://www.wikidata.org/entity/Q27485157
mfk,http://www.wikidata.org/entity/Q1954633
mgb,http://www.wikidata.org/entity/Q3330578
mgz,http://www.wikidata.org/entity/Q668213
mhl,http://www.wikidata.org/entity/Q3330589
mov,http://www.wikidata.org/entity/Q98371215
mws,http://www.wikidata.org/entity/Q961468
raf,http://www.wikidata.org/entity/Q2126295
rkk,http://www.wikidata.org/entity/Q98356453
rra,http://www.wikidata.org/entity/Q2126159
rti,http://www.wikidata.org/entity/Q114794
rtr,http://www.wikidata.org/entity/Q688243
rts,http://www.wikidata.org/entity/Q3416973
rxb,http://www.wikidata.org/entity/Q1025578
sap,http://www.wikidata.org/entity/Q50920401
sbb,http://www.wikidata.org/entity/Q155336
sik,http://www.wikidata.org/entity/Q2256716
snb,http://www.wikidata.org/entity/Q201787
snp,http://www.wikidata.org/entity/Q2256277
son,http://www.wikidata.org/entity/Q98380014
soz,http://www.wikidata.org/entity/Q2256734
srf,http://www.wikidata.org/entity/Q27490292
sts,http://www.wikidata.org/entity/Q27488773
swi,http://www.wikidata.org/entity/Q568173
ubb,http://www.wikidata.org/entity/Q683842
vks,http://www.wikidata.org/entity/Q670595
zbz,http://www.wikidata.org/entity/Q190260
zem,http://www.wikidata.org/entity/Q28500447
\ No newline at end of file
src/test/resources/enrichement/coordinates_for_testing_lat_long.csv 0 → 100644
View file @ ffad8b46
"aag","47.38787196, 8.049455603"
"abe","46.95089, 7.43706"
"acj","47.41728, 7.07422"
"adg","46.8542, 9.5382"
"afz","47.375219, 8.545961"
"agl","47.04153, 9.06705"
"apf","46.5199, 6.6332"
"atd","46.186235, 8.732027"
"ati","46.192846, 9.0132"
"avl","46.52511, 6.62441"
"azh","47.39077, 8.512219"
"baa","45.86725, 8.98326"
"bab","47.551869, 7.589958"
"bar","46.940555555, 7.446388888"
"baz","47.37237, 8.54582"
"bbb","46.9473, 7.4483"
"bcf","46.8066, 7.15597"
"bfl","46.92833333, 7.45166667"
"bmf","47.5567059, 7.578635"
"bpu","46.9902, 6.9311"
"bvc","47.1042, 6.82634"
"cde","46.19309, 9.01245"
"cdt","46.19908, 6.13594"
"cag","46.1992, 6.1376"
"cic","46.2274, 6.1373"
"clg","46.87994, 8.64185"
"clu","47.10016, 6.8239"
"csa","46.60208, 6.53612"
"fad","46.44019, 8.93677"
"fer","46.1314399, 8.802462"
"fgr","46.85014, 9.53323"
"fpc","46.194709, 9.024187"
"fss","47.4958, 8.7383"
"gvs","46.88057, 8.64469"
"hgk","47.53307, 7.61098"
"hsl","47.07143, 8.27772"
"ias","46.52378, 6.58423"
"ikg","46.85111, 9.533846"
"ikr","46.22169, 6.12565"
"kak","47.3941882, 8.0587246"
"kbg","46.849522222, 9.533855555"
"kek","47.12683, 8.75293"
"khz","47.370278, 8.548056"
"klu","47.04138, 8.310911"
"kmm","47.06376, 7.09299"
"lfg","47.11458, 8.38545"
"lkb","46.94049, 7.44218"
"lmz","47.379166666, 8.539722222"
"maa","47.3944147, 8.0452584"
"mav","46.938154, 7.394621"
"mcl","46.0043, 8.95327"
"mdl","46.38, 6.24018"
"meg","46.197797222, 6.137313888"
"mel","46.509824, 6.632767"
"mem","46.1736, 8.81054"
"mfk","46.94186, 7.45004"
"mgb","46.6167, 7.058596"
"mgz","47.39077, 8.512219"
"mhl","46.5221, 6.63491"
"mov","46.94134, 7.436"
"mws","46.23168, 7.35853"
"raf","47.50045, 8.72527"
"rkk","47.37968, 8.52745"
"rra","47.37845, 8.52948"
"rti","46.00373, 8.9512"
"rtr","46.8520381, 9.5344336"
"rts","46.2, 6.083333333"
"rxb","47.53496, 7.59477"
"sap","46.949135, 7.436426111"
"sbb","47.47540096, 8.205857926"
"sik","47.362485, 8.555264"
"snb","46.941444, 7.449667"
"snp","46.00605, 8.9399"
"son","46.94814, 7.45241"
"soz","47.366827777, 8.547530555"
"srf","47.40146, 8.53547"
"sts","47.6973, 8.6337185"
"swi","46.943487611, 7.473449888"
"ubb","47.5594, 7.5812"
"vks","47.052777777, 8.335833333"
"zbz","47.374166666, 8.545277777"
"zem","46.96684, 7.45468"
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment