In order to mitigate against the brute force attacks against Gitlab accounts, we are moving to all edu-ID Logins. We would like to remind you to link your account with your edu-id. Login will be possible only by edu-ID after November 30, 2021. Here you can find the instructions for linking your account.

If you don't have a SWITCH edu-ID, you can create one with this guide here

kind regards

This Server has been upgraded to GitLab release 14.2.6

RdfTransformer.kt 12.1 KB
Newer Older
Jonas Waeber's avatar
Jonas Waeber committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
/*
 * Drupal Sync Service
 * Copyright (C) 2020 Memoriav
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */

Jonas Waeber's avatar
Jonas Waeber committed
19
20
package org.memobase

21
22
23
24
25
26
27
import ch.memobase.rdf.MB
import ch.memobase.rdf.NS
import ch.memobase.rdf.RDA
import ch.memobase.rdf.RDF
import ch.memobase.rdf.RICO
import ch.memobase.rdf.SCHEMA
import ch.memobase.rdf.WD
28
import java.util.Properties
29
30
31
import org.apache.jena.rdf.model.Literal
import org.apache.jena.rdf.model.Model
import org.apache.jena.rdf.model.ModelFactory
Jonas Waeber's avatar
Jonas Waeber committed
32
33
import org.apache.jena.rdf.model.Property
import org.apache.jena.rdf.model.RDFNode
34
35
import org.apache.jena.rdf.model.Resource
import org.apache.jena.rdf.model.ResourceFactory
Jonas Waeber's avatar
Jonas Waeber committed
36
import org.apache.logging.log4j.LogManager
Jonas Waeber's avatar
Jonas Waeber committed
37
38
39
import org.memobase.model.Address
import org.memobase.model.Institution
import org.memobase.model.RecordSet
40
import org.memobase.model.RichText
Jonas Waeber's avatar
Jonas Waeber committed
41

42
43
class RdfTransformer(properties: Properties) {
    private val log = LogManager.getLogger("RdfTransformer")
Jonas Waeber's avatar
Jonas Waeber committed
44

45
46
47
    private val cantons = Util.getCantons()
    private val municipalities = Util.getMunicipalities()
    private val languages = Util.loadLanguages(properties.getProperty(Util.languageSourceFilePathPropertyName))
Jonas Waeber's avatar
Jonas Waeber committed
48
49
50
51
52

    fun createInstitution(input: Institution): Pair<String, Model> {
        val model = ModelFactory.createDefaultModel()

        val resource = model.createResource(NS.mbcb + input.field_memobase_id)
53
54
        resource.addProperty(RDF.type, RICO.CorporateBody)
        resource.addProperty(RICO.type, "memobaseInstitution")
Jonas Waeber's avatar
Jonas Waeber committed
55
        resource.addLiteral(MB.isPublished, input.status)
Jonas Waeber's avatar
Jonas Waeber committed
56

57
58
59
        resource.addProperty(RICO.identifiedBy, addIdentifier(model, "main", input.field_memobase_id))
        if (input.field_old_memobase_id != null)
            resource.addProperty(RICO.identifiedBy, addIdentifier(model, "oldMemobase", input.field_old_memobase_id))
Jonas Waeber's avatar
Jonas Waeber committed
60

Jonas Waeber's avatar
Jonas Waeber committed
61
62
63
64
        input.recordset_ids.forEach {
            resource.addProperty(RICO.isHolderOf, NS.mbrs + it)
        }

Jonas Waeber's avatar
Jonas Waeber committed
65
66
67
        resource.addProperty(RICO.name, langLiteral(input.title, "de"))
        resource.addProperty(RICO.name, langLiteral(input.title_fr, "fr"))
        resource.addProperty(RICO.name, langLiteral(input.title_it, "it"))
68
69
70
        addIfNotNull(resource, input.field_text, "de")
        addIfNotNull(resource, input.field_text_fr, "fr")
        addIfNotNull(resource, input.field_text_it, "it")
Jonas Waeber's avatar
Jonas Waeber committed
71

Jonas Waeber's avatar
Jonas Waeber committed
72
        input.field_address.forEach { address ->
73
            val location = generateLocationResource(model, address)
Jonas Waeber's avatar
Jonas Waeber committed
74
            resource.addProperty(RICO.hasLocation, location)
Jonas Waeber's avatar
Jonas Waeber committed
75
        }
Jonas Waeber's avatar
Jonas Waeber committed
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
        input.field_isil.let {
            if (it != null)
                resource.addProperty(WD.isil, literal(it))
        }
        input.field_email.let {
            if (it != null) {
                resource.addProperty(WD.emailAddress, literal(it))
            }
        }
        input.field_website.let {
            if (it != null)
                resource.addProperty(WD.website, literal(it.uri))
        }
        input.field_link_archive_catalog.let {
            if (it != null)
                resource.addProperty(WD.onlineArchive, literal(it.uri))
        }
Jonas Waeber's avatar
Jonas Waeber committed
93
94
95
        input.field_institution_types.forEach {
            resource.addProperty(WD.typeOfInstitution, model.createResource(NS.wd + it.substringAfterLast("/")))
        }
96
97
        if (input.computed_teaser_image_url != null)
            resource.addLiteral(WD.image, input.computed_teaser_image_url)
Jonas Waeber's avatar
Jonas Waeber committed
98
        return Pair(resource.uri, model)
Jonas Waeber's avatar
Jonas Waeber committed
99
100
    }

Jonas Waeber's avatar
Jonas Waeber committed
101
    fun createRecordSet(input: RecordSet): Pair<String, Model> {
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
        val model = ModelFactory.createDefaultModel()

        val resource = model.createResource(NS.mbrs + input.field_memobase_id)
        resource.addProperty(RDF.type, RICO.RecordSet)
        resource.addLiteral(MB.isPublished, input.status)

        resource.addProperty(RICO.identifiedBy, addIdentifier(model, "main", input.field_memobase_id))
        if (input.field_old_memobase_id != null)
            resource.addProperty(RICO.identifiedBy, addIdentifier(model, "oldMemobase", input.field_old_memobase_id))

        input.field_institution.forEach {
            resource.addProperty(RICO.heldBy, NS.mbcb + it)
        }

        resource.addProperty(RICO.title, langLiteral(input.title, "de"))
        resource.addProperty(RICO.title, langLiteral(input.title_fr, "fr"))
        resource.addProperty(RICO.title, langLiteral(input.title_it, "it"))
119
120
121
122

        addIfNotNull(resource, input.field_text, "de")
        addIfNotNull(resource, input.field_text_fr, "fr")
        addIfNotNull(resource, input.field_text_it, "it")
123
124
125
126
127
128
129
130
131
132
133
134

        input.field_metadata_language_codes.forEach {
            // rico:hasLanguage metadata
            resource.addProperty(RICO.hasLanguage, addLanguage(model, it))
        }
        // rico:hasTitle main
        resource.addProperty(RICO.hasTitle, addTitle(model, input.title, input.title_fr, input.title_it))
        resource.addProperty(RDA.hasSponsoringAgentOfResource, model.createResource(Util.memoriavUri))

        if (input.computed_teaser_image_url != null)
            resource.addProperty(WD.image, literal(input.computed_teaser_image_url))

Jonas Waeber's avatar
Jonas Waeber committed
135
136
137
138
139
140
141
142
143
        input.field_time_period.let {
            if (it != null) {
                val date = model.createResource()
                date.addProperty(RDF.type, RICO.DateRange)
                date.addProperty(RICO.normalizedDateValue, it)
                resource.addProperty(RICO.isAssociatedWithDate, date)
            }
        }

144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
        // rico:scopeAndContent -> Inhalt
        // rico:history -> Kontext
        // rico:integrity -> Auswahl / Vollständigkeit
        // dct:conformsTo -> Informationen zur Erschliessung
        // rico:Identifier callNumber -> Original-Signatur des Bestands
        // dct:created
        // rico:type -> Dokumenttyp
        // rico:descriptiveNote -> Datenübernahme

        // rico:Title original

        // rico:isRecordResourceAssociatedWithRecordResource -> Dokumente rico:Record ( sameAs + title) | Verwandte Bestände rico:RecordSet
        // Originale Bestandesbeschreibung -> rico:RecordSet -> rico:hasSource

        // rico:recordResourceOrInstantiationIsTargetOfRecordResourceHoldingRelation
        // rico:recordResourceExtent -> Umfang
        // rico:isSubjectOf -> publications rico:Record (sameAs + title)

        // rico:conditionsOfUse -> Rechte
        // rico:conditionsOfAccess -> Zugang
        // rico:publicationDate -> Datum der Übernahme in Memobase
        // rico:modificationDate (is enriched when indexed, is that enough? Should I do this here?).

        // hasTitle original-> Originaltitel des Bestands
        // schema:sameAs -> Projektname und Beschreibung | Publikationen

        // rdau:P60099 has language of resource -> What is this supposed to do?
        // rdau:P60470 has note on resource -> Hinweise
        // rdau:P60848 has referential resource relationship with


        return Pair(resource.uri, model)
    }

    private fun addIdentifier(model: Model, type: String, value: String): Resource {
        val identifier = model.createResource()
        identifier.addProperty(RDF.type, RICO.Identifier)
        identifier.addProperty(RICO.type, literal(type))
        identifier.addProperty(RICO.identifier, literal(value))
        return identifier
    }

    // TODO: Add language translations.
    private fun addLanguage(model: Model, value: String): Resource {
        val language = model.createResource()
        language.addProperty(RDF.type, RICO.Language)
        language.addProperty(RICO.type, literal("metadata"))
        languages[value].let {
            if (it == null) {
                language.addProperty(RICO.name, literal(value))
            } else {
                language.addProperty(RICO.name, langLiteral(it.de, "de"))
                language.addProperty(RICO.name, langLiteral(it.fr, "fr"))
                language.addProperty(RICO.name, langLiteral(it.it, "it"))
                language.addProperty(SCHEMA.sameAs, literal(it.id))
            }
        }
        return language
    }

    private fun addTitle(model: Model, de: String, fr: String, it: String): Resource {
        val language = model.createResource()
        language.addProperty(RDF.type, RICO.Title)
        language.addProperty(RICO.type, literal("main"))
        language.addProperty(RICO.title, langLiteral(de, "de"))
        language.addProperty(RICO.title, langLiteral(fr, "fr"))
        language.addProperty(RICO.title, langLiteral(it, "it"))
        return language
Jonas Waeber's avatar
Jonas Waeber committed
212
213
    }

214
215
    private fun generateLocationResource(model: Model, address: Address): Resource {
        val location = model.createResource()
216

Jonas Waeber's avatar
Jonas Waeber committed
217
218
219
220
221
222
        val streetAddress = address.address_line1
        val secondAddressLine = address.address_line2
        val combinedStreetAddress = if (secondAddressLine.isNullOrEmpty()) {
            streetAddress
        } else {
            streetAddress + "\n" + secondAddressLine
Jonas Waeber's avatar
Jonas Waeber committed
223
        }
Jonas Waeber's avatar
Jonas Waeber committed
224
225
226
227
228
229
        val streetNumber = streetAddress.substringAfterLast(" ")
        val street = streetAddress.replace(streetNumber, "").trim()
        location.addProperty(WD.street, literal(street))
        location.addProperty(WD.streetNumber, literal(streetNumber))
        location.addProperty(WD.streetAddress, literal(combinedStreetAddress))

Jonas Waeber's avatar
Jonas Waeber committed
230

Jonas Waeber's avatar
Jonas Waeber committed
231
        val postalCode = address.postal_code.trim()
Jonas Waeber's avatar
Jonas Waeber committed
232

233
        location.addProperty(RDF.type, RICO.Place)
Jonas Waeber's avatar
Jonas Waeber committed
234
        location.addProperty(WD.postalCode, literal(postalCode))
Jonas Waeber's avatar
Jonas Waeber committed
235
236
237

        location.addProperty(WD.coordinates, literal(address.coordinates))

238
        val canton = model.createResource()
Jonas Waeber's avatar
Jonas Waeber committed
239
240
241
242
243
244
245
246
247
248
249
250
251
        canton.addProperty(RDF.type, RICO.Place)
        location.addProperty(WD.adminUnit, canton)
        cantons[address.administrative_area].let {
            // this should always be the case!
            if (it != null) {
                canton.addProperty(RICO.name, langLiteral(it.de, "de"))
                canton.addProperty(RICO.name, langLiteral(it.fr, "fr"))
                canton.addProperty(RICO.name, langLiteral(it.it, "it"))
                canton.addProperty(SCHEMA.sameAs, it.id)
            } else {
                // this shouldn't happen!
                canton.addProperty(RICO.name, literal("Unknown"))
            }
252
            canton.addProperty(RICO.type, literal("canton"))
Jonas Waeber's avatar
Jonas Waeber committed
253
254
        }

255
        val municipality = model.createResource()
Jonas Waeber's avatar
Jonas Waeber committed
256
257
258
259
260
261
262
263
264
265
        municipality.addProperty(RDF.type, RICO.Place)
        location.addProperty(WD.adminUnit, municipality)
        municipalities[postalCode].let {
            if (it != null) {
                municipality.addProperty(RICO.name, langLiteral(it.de, "de"))
                municipality.addProperty(RICO.name, langLiteral(it.fr, "fr"))
                municipality.addProperty(RICO.name, langLiteral(it.it, "it"))
                municipality.addProperty(SCHEMA.sameAs, it.id)
            } else {
                municipality.addProperty(RICO.name, literal(address.locality))
Jonas Waeber's avatar
Jonas Waeber committed
266
            }
267
            municipality.addProperty(RICO.type, "municipality")
Jonas Waeber's avatar
Jonas Waeber committed
268
269
270
        }
        // country is currently hard coded to switzerland!
        location.addProperty(WD.country, WD.switzerland)
Jonas Waeber's avatar
Jonas Waeber committed
271
272
273
        return location
    }

Jonas Waeber's avatar
Jonas Waeber committed
274
275
    private fun langLiteral(text: String, language: String): Literal =
        ResourceFactory.createLangLiteral(text.trim(), language)
276

Jonas Waeber's avatar
Jonas Waeber committed
277
    private fun literal(text: String): Literal = ResourceFactory.createPlainLiteral(text.trim())
278
279
280
281
282
283
284

    private fun addIfNotNull(resource: Resource, field: RichText?, language: String) {
        field.let {
            if (it != null)
                resource.addProperty(RICO.descriptiveNote, langLiteral(it.value, language))
        }
    }
Jonas Waeber's avatar
Jonas Waeber committed
285
286
287
288
289
290
291

    private fun addIfNotNull(resource: Resource, field: String?, property: Property, node: RDFNode) {
        field.let {
            if (it != null)
                resource.addProperty(property, node)
        }
    }
Jonas Waeber's avatar
Jonas Waeber committed
292
}