Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
memoriav
Memobase 2020
services
Elasticsearch Services
Search Doc Service
Commits
f2f4c8cb
Commit
f2f4c8cb
authored
Sep 09, 2020
by
Jonas Waeber
Browse files
Remove namespaces for institution & record set facet values
parent
1ca0bf50
Pipeline
#14007
passed with stages
in 5 minutes and 32 seconds
Changes
7
Pipelines
2
Expand all
Hide whitespace changes
Inline
Side-by-side
build.gradle
View file @
f2f4c8cb
...
...
@@ -32,7 +32,7 @@ ext {
}
dependencies
{
implementation
'org.memobase:memobase-service-utilities:
1.6.0
'
implementation
'org.memobase:memobase-service-utilities:
0.14.2
'
// Logging Framework
implementation
"org.apache.logging.log4j:log4j-api:${log4jV}"
...
...
src/main/kotlin/KEYS.kt
View file @
f2f4c8cb
...
...
@@ -19,9 +19,11 @@
package
org.memobase
object
KEYS
{
const
val
isPublished
=
"isPublished"
const
val
entityId
=
"@id"
const
val
atType
=
"@type"
const
val
isPublished
=
"isPublished"
const
val
ricoType
=
"type"
const
val
firstName
=
"firstName"
const
val
lastName
=
"lastName"
...
...
@@ -36,6 +38,9 @@ object KEYS {
const
val
producer
=
"P60441"
const
val
publishedBy
=
"publishedBy"
const
val
isPartOf
=
"isPartOf"
const
val
heldBy
=
"heldBy"
const
val
contributor
=
"contributor"
const
val
creator
=
"creator"
...
...
src/main/kotlin/SearchDocTransform.kt
View file @
f2f4c8cb
...
...
@@ -130,16 +130,8 @@ class SearchDocTransform {
sameAs
=
Extract
.
listOfStrings
(
record
[
"sameAs"
]),
abstract
=
Extract
.
languageContainer
(
"abstract"
,
record
[
"abstract"
]),
id
=
id
,
institution
=
listOf
(
FacettedContainer
(
LanguageContainer
(
emptyList
(),
emptyList
(),
emptyList
(),
emptyList
()),
listOf
(
record
[
"heldBy"
]
as
String
)
)
),
recordSet
=
FacettedContainer
(
LanguageContainer
(
emptyList
(),
emptyList
(),
emptyList
(),
emptyList
()),
listOf
(
record
[
"isPartOf"
]
as
String
)
),
institution
=
Meta
.
extractInstitution
(
record
),
recordSet
=
Meta
.
extractRecordSet
(
record
)[
0
],
descriptiveNote
=
Extract
.
languageContainer
(
"descriptiveNote"
,
record
[
"descriptiveNote"
]),
scopeAndContent
=
Extract
.
languageContainer
(
"scopeAndContent"
,
record
[
"scopeAndContent"
]),
relatedMaterial
=
Extract
.
languageContainer
(
"relation"
,
record
[
"relation"
]),
...
...
src/main/kotlin/helpers/Meta.kt
0 → 100644
View file @
f2f4c8cb
package
org.memobase.helpers
import
com.beust.klaxon.JsonObject
import
org.apache.logging.log4j.LogManager
import
org.memobase.KEYS
import
org.memobase.model.FacettedContainer
import
org.memobase.model.LanguageContainer
import
org.memobase.rdf.NS
object
Meta
{
private
val
log
=
LogManager
.
getLogger
(
"InstitutionExtraction"
)
fun
extractInstitution
(
record
:
JsonObject
):
List
<
FacettedContainer
>
{
return
extract
(
KEYS
.
heldBy
,
record
)
}
fun
extractRecordSet
(
record
:
JsonObject
):
List
<
FacettedContainer
>
{
return
extract
(
KEYS
.
isPartOf
,
record
)
}
// TODO: Implement function to extract labels from elastic index.
private
fun
extract
(
key
:
String
,
record
:
JsonObject
):
List
<
FacettedContainer
>
{
return
record
[
key
].
let
{
items
->
when
(
items
)
{
is
String
->
listOf
(
FacettedContainer
(
LanguageContainer
(
emptyList
(),
emptyList
(),
emptyList
(),
emptyList
()),
listOf
(
removeNamespace
(
key
,
items
))
)
)
is
List
<
*
>
->
items
.
map
{
item
->
FacettedContainer
(
LanguageContainer
(
emptyList
(),
emptyList
(),
emptyList
(),
emptyList
()),
listOf
(
removeNamespace
(
key
,
item
as
String
))
)
}
else
->
{
log
.
error
(
"Could not extract institution in property heldBy from record ${record[KEYS.entityId]}"
)
emptyList
()
}
}
}
}
private
fun
removeNamespace
(
key
:
String
,
value
:
String
):
String
{
return
when
(
key
)
{
KEYS
.
isPartOf
->
value
.
removePrefix
(
NS
.
mbrs
)
KEYS
.
heldBy
->
value
.
removePrefix
(
NS
.
mbcb
)
else
->
value
}
}
}
\ No newline at end of file
src/test/resources/data/1/input.json
View file @
f2f4c8cb
This diff is collapsed.
Click to expand it.
src/test/resources/data/1/output.json
View file @
f2f4c8cb
{
"title"
:[{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Bern: Altstadt, obere; Bundesplatz 4; Bärenplatz (31); Käfiggässchen; -- Fuhrwerk; Automobil; Lastwagen; Transport, Verkehr; Strassenbeleuchtung"
]}],
"type"
:
"Foto"
,
"sourceID"
:
"216133"
,
"id"
:
"https://memobase.ch/record/Burgerbib-Krebser-216133"
,
"descriptiveNote"
:[{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Datierung: Bundesplatz / Büren-Besitzung<br>"
]}],
"rightsHolder"
:[{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Burgerbibliothek Bern"
]}],
"sameAs"
:[
"http://katalog.burgerbib.ch/detail.aspx?ID=216133"
],
"keywords"
:[{
"name"
:{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Transport, Verkehr"
]},
"facet"
:[
"Transport, Verkehr"
]}],
"personCreator"
:[{
"name"
:{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Anonym"
]},
"relation"
:{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"author"
]},
"facet"
:[
"0~A~#"
,
"1~A~Anonym~#"
]}],
"personContributor"
:[{
"name"
:{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Familie Krebser"
]},
"relation"
:{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Collector/Sammler"
]},
"facet"
:[
"0~F~#"
,
"1~F~Familie Krebser~#"
]}],
"personsFacet"
:[
"0~A~#"
,
"1~A~Anonym~#"
,
"0~F~#"
,
"1~F~Familie Krebser~#"
],
"placeRelated"
:[{
"name"
:{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Käfiggässchen"
]},
"facet"
:[
"0~K~#"
,
"1~K~Käfiggässchen~#"
]}],
"placeFacet"
:[
"0~K~#"
,
"1~K~Käfiggässchen~#"
],
"dateCreated"
:[{
"date"
:
"1900/1909"
,
"facet"
:[]}],
"institution"
:[{
"name"
:{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[]},
"facet"
:[
"https://memobase.ch/institution/Burgerbib"
]}],
"recordSet"
:{
"name"
:{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[]},
"facet"
:[
"https://memobase.ch/recordSet/Burgerbib-Krebser"
]},
"memoriavClaim"
:
true
,
"callNumber"
:[
"Historische Sammlung Krebser 13/2"
],
"physicalCharacteristics"
:[{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Weite: 85 mm"
,
"Höhe: 100 mm"
]}],
"format"
:[{
"name"
:{
"de"
:[
"Glasplatte"
],
"fr"
:[],
"it"
:[],
"un"
:[]},
"facet"
:[
"http://www.wikidata.org/entity/Q1138868"
]},{
"name"
:{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Glasplatte"
]},
"facet"
:[]}],
"published"
:
false
,
"suggest"
:{
"title"
:[
"Bern: Altstadt, obere; Bundesplatz 4; Bärenplatz (31); Käfiggässchen; -- Fuhrwerk; Automobil; Lastwagen; Transport, Verkehr; Strassenbeleuchtung"
],
"seriesTitle"
:[],
"broadcastTitle"
:[],
"keywords"
:[
"Transport, Verkehr"
]}}
\ No newline at end of file
{
"title"
:[{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Bern: Altstadt, obere; Bundesplatz 4; Bärenplatz (31); Käfiggässchen; -- Fuhrwerk; Automobil; Lastwagen; Transport, Verkehr; Strassenbeleuchtung"
]}],
"type"
:
"Foto"
,
"sourceID"
:
"216133"
,
"id"
:
"https://memobase.ch/record/Burgerbib-Krebser-216133"
,
"descriptiveNote"
:[{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Datierung: Bundesplatz / Büren-Besitzung<br>"
]}],
"rightsHolder"
:[{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Burgerbibliothek Bern"
]}],
"sameAs"
:[
"http://katalog.burgerbib.ch/detail.aspx?ID=216133"
],
"keywords"
:[{
"name"
:{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Transport, Verkehr"
]},
"facet"
:[
"Transport, Verkehr"
]}],
"personCreator"
:[{
"name"
:{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Anonym"
]},
"relation"
:{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"author"
]},
"facet"
:[
"0~A~#"
,
"1~A~Anonym~#"
]}],
"personContributor"
:[{
"name"
:{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Familie Krebser"
]},
"relation"
:{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Collector/Sammler"
]},
"facet"
:[
"0~F~#"
,
"1~F~Familie Krebser~#"
]}],
"personsFacet"
:[
"0~A~#"
,
"1~A~Anonym~#"
,
"0~F~#"
,
"1~F~Familie Krebser~#"
],
"placeRelated"
:[{
"name"
:{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Käfiggässchen"
]},
"facet"
:[
"0~K~#"
,
"1~K~Käfiggässchen~#"
]}],
"placeFacet"
:[
"0~K~#"
,
"1~K~Käfiggässchen~#"
],
"dateCreated"
:[{
"date"
:
"1900/1909"
,
"facet"
:[]}],
"institution"
:[{
"name"
:{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[]},
"facet"
:[
"Burgerbib"
]}],
"recordSet"
:{
"name"
:{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[]},
"facet"
:[
"Burgerbib-Krebser"
]},
"memoriavClaim"
:
true
,
"callNumber"
:[
"Historische Sammlung Krebser 13/2"
],
"physicalCharacteristics"
:[{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Weite: 85 mm"
,
"Höhe: 100 mm"
]}],
"format"
:[{
"name"
:{
"de"
:[
"Glasplatte"
],
"fr"
:[],
"it"
:[],
"un"
:[]},
"facet"
:[
"http://www.wikidata.org/entity/Q1138868"
]},{
"name"
:{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Glasplatte"
]},
"facet"
:[]}],
"published"
:
false
,
"suggest"
:{
"title"
:[
"Bern: Altstadt, obere; Bundesplatz 4; Bärenplatz (31); Käfiggässchen; -- Fuhrwerk; Automobil; Lastwagen; Transport, Verkehr; Strassenbeleuchtung"
],
"seriesTitle"
:[],
"broadcastTitle"
:[],
"keywords"
:[
"Transport, Verkehr"
]}}
\ No newline at end of file
src/test/resources/data/2/output.json
View file @
f2f4c8cb
{
"title"
:[{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Anonym -- Armut"
]}],
"type"
:
"Foto"
,
"sourceID"
:
"208576"
,
"id"
:
"https://memobase.ch/record/Burgerbib-Krebser-208576"
,
"abstract"
:[{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Bildeintrag: auf der Verpackung: 49543<br>"
]}],
"descriptiveNote"
:[{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Datierung: Negativnummer<br>"
]}],
"rightsHolder"
:[{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Burgerbibliothek Bern"
]}],
"sameAs"
:[
"http://katalog.burgerbib.ch/detail.aspx?ID=208576"
],
"keywords"
:[{
"name"
:{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Armut"
]},
"facet"
:[
"Armut"
]}],
"personCreator"
:[{
"name"
:{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Moeglé, Jean"
]},
"relation"
:{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"author"
]},
"facet"
:[
"0~M~#"
,
"1~M~Moeglé, Jean~#"
]}],
"personContributor"
:[{
"name"
:{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Familie Krebser"
]},
"relation"
:{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Collector/Sammler"
]},
"facet"
:[
"0~F~#"
,
"1~F~Familie Krebser~#"
]}],
"personsFacet"
:[
"0~F~#"
,
"1~F~Familie Krebser~#"
,
"0~M~#"
,
"1~M~Moeglé, Jean~#"
],
"dateCreated"
:[{
"date"
:
"20. Jh."
,
"facet"
:[]}],
"institution"
:[{
"name"
:{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[]},
"facet"
:[
"https://memobase.ch/institution/Burgerbib"
]}],
"recordSet"
:{
"name"
:{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[]},
"facet"
:[
"https://memobase.ch/recordSet/Burgerbib-Krebser"
]},
"memoriavClaim"
:
true
,
"callNumber"
:[
"Historische Sammlung Krebser 64/21"
],
"physicalCharacteristics"
:[{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Weite: 165 mm"
,
"Höhe: 120 mm"
]}],
"format"
:[{
"name"
:{
"de"
:[
"Glasplatte"
],
"fr"
:[],
"it"
:[],
"un"
:[]},
"facet"
:[
"http://www.wikidata.org/entity/Q1138868"
]},{
"name"
:{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Glasplatte"
]},
"facet"
:[]}],
"published"
:
false
,
"suggest"
:{
"title"
:[
"Anonym -- Armut"
],
"seriesTitle"
:[],
"broadcastTitle"
:[],
"keywords"
:[
"Armut"
]}}
\ No newline at end of file
{
"title"
:[{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Anonym -- Armut"
]}],
"type"
:
"Foto"
,
"sourceID"
:
"208576"
,
"id"
:
"https://memobase.ch/record/Burgerbib-Krebser-208576"
,
"abstract"
:[{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Bildeintrag: auf der Verpackung: 49543<br>"
]}],
"descriptiveNote"
:[{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Datierung: Negativnummer<br>"
]}],
"rightsHolder"
:[{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Burgerbibliothek Bern"
]}],
"sameAs"
:[
"http://katalog.burgerbib.ch/detail.aspx?ID=208576"
],
"keywords"
:[{
"name"
:{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Armut"
]},
"facet"
:[
"Armut"
]}],
"personCreator"
:[{
"name"
:{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Moeglé, Jean"
]},
"relation"
:{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"author"
]},
"facet"
:[
"0~M~#"
,
"1~M~Moeglé, Jean~#"
]}],
"personContributor"
:[{
"name"
:{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Familie Krebser"
]},
"relation"
:{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Collector/Sammler"
]},
"facet"
:[
"0~F~#"
,
"1~F~Familie Krebser~#"
]}],
"personsFacet"
:[
"0~F~#"
,
"1~F~Familie Krebser~#"
,
"0~M~#"
,
"1~M~Moeglé, Jean~#"
],
"dateCreated"
:[{
"date"
:
"20. Jh."
,
"facet"
:[]}],
"institution"
:[{
"name"
:{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[]},
"facet"
:[
"Burgerbib"
]}],
"recordSet"
:{
"name"
:{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[]},
"facet"
:[
"Burgerbib-Krebser"
]},
"memoriavClaim"
:
true
,
"callNumber"
:[
"Historische Sammlung Krebser 64/21"
],
"physicalCharacteristics"
:[{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Weite: 165 mm"
,
"Höhe: 120 mm"
]}],
"format"
:[{
"name"
:{
"de"
:[
"Glasplatte"
],
"fr"
:[],
"it"
:[],
"un"
:[]},
"facet"
:[
"http://www.wikidata.org/entity/Q1138868"
]},{
"name"
:{
"de"
:[],
"fr"
:[],
"it"
:[],
"un"
:[
"Glasplatte"
]},
"facet"
:[]}],
"published"
:
false
,
"suggest"
:{
"title"
:[
"Anonym -- Armut"
],
"seriesTitle"
:[],
"broadcastTitle"
:[],
"keywords"
:[
"Armut"
]}}
\ No newline at end of file
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment