In order to mitigate against the brute force attacks against Gitlab accounts, we are moving to all edu-ID Logins. We would like to remind you to link your account with your edu-id. Login will be possible only by edu-ID after November 30, 2021. Here you can find the instructions for linking your account.

If you don't have a SWITCH edu-ID, you can create one with this guide here

kind regards

This Server has been upgraded to GitLab release 14.2.6

Extract.kt 7.46 KB
Newer Older
Jonas Waeber's avatar
Jonas Waeber committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
/*
 * search-doc-service
 * Copyright (C) 2020 Memoriav
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */

package org.memobase.helpers

import com.beust.klaxon.JsonArray
import com.beust.klaxon.JsonObject
import org.apache.logging.log4j.LogManager
import org.memobase.model.CreatorNameContainer
import org.memobase.model.DateContainer
import org.memobase.model.LanguageContainer
import org.memobase.model.NameContainer

object Extract {
    private val log = LogManager.getLogger("ExtractionHelper")

    private fun extractLanguageTag(entity: JsonObject, language: String): List<String> {
        return entity["@value"].let {
            if (entity["@language"] == language && it != null)
                listOf(it as String)
            else
                emptyList()
        }
    }

    fun extractLanguageContainer(parent: String, entity: Any?): List<LanguageContainer> {
        return when (entity) {
            is String -> listOf(LanguageContainer(emptyList(), emptyList(), emptyList(), listOf(entity)))
            is JsonObject ->
                listOf(
                    LanguageContainer(
                        de = extractLanguageTag(entity, "de"),
                        fr = extractLanguageTag(entity, "fr"),
                        it = extractLanguageTag(entity, "it"),
                        un = emptyList()
                    )
                )
            is List<*> ->
                listOf(entity.mapNotNull { subEntity ->
                    when (subEntity) {
                        is String ->
                            LanguageContainer(emptyList(), emptyList(), emptyList(), listOf(subEntity))
                        is JsonObject ->
                            LanguageContainer(
                                de = extractLanguageTag(subEntity, "de"),
                                fr = extractLanguageTag(subEntity, "fr"),
                                it = extractLanguageTag(subEntity, "it"),
                                un = emptyList()
                            )
                        else -> {
                            log.error("Could not extract language container from subEntity $subEntity in entity $entity.")
                            null
                        }
                    }
                }.reduce { acc, languageContainer ->
                    acc.merge(languageContainer)
                })
            else -> {
                log.error("Could not extract language container from entity: $entity in parent $parent.")
                emptyList()
            }
        }
    }

    fun extractDate(entities: List<JsonObject>): List<DateContainer> {
        return entities.map { entity ->
            val isNormalized = entity.containsKey("rico:normalizedDateValue")
            val date = if (isNormalized) {
                entity["rico:normalizedDateValue"] as String
            } else {
                entity["rico:expressedDate"] as String
            }
            val qualifier = entity["rico:dateQualifier"] as String?
            val certainty = entity["rico:certainty"] as String?
            val facetList = when (entity["@type"] as String) {
                "rico:SingleDate" ->
                    if (isNormalized)
                        DateFacetBuilder.buildFromNormalizedSingleDate(date)
                    else emptyList()
                "rico:DateRange" ->
                    if (isNormalized)
                        DateFacetBuilder.buildFromNormalizedDateRange(date)
                    else emptyList()
                else -> emptyList()
            }
            DateContainer(
                date = date,
                qualifier = qualifier,
                certainty = certainty,
                facet = facetList
            )
        }
    }

    fun typedEntityByType(
        entities: List<JsonObject>,
        field: String,
        type: String,
        annotationField: String
    ): List<LanguageContainer> {
        return entities.filter { it[field] == type }.flatMap {
            extractLanguageContainer(it["@id"] as String, it[annotationField])
        }
    }

    fun facetEntity(entities: List<JsonObject>, property: String): List<NameContainer> {
        return entities.mapNotNull {
            val lang = extractLanguageContainer(it["@id"] as String, it[property])
            if (lang.isEmpty()) {
                null
            } else {
                NameContainer(lang.first(), lang.first().toList())
            }
        }
    }

    fun extractPlaces(entities: List<JsonObject>): List<LanguageContainer> {
        return entities.flatMap {
            extractLanguageContainer(it["@id"] as String, it["rico:name"])
        }
    }

    fun extractSourceId(entities: List<JsonObject>): String {
        return entities.first {
            it["rico:type"] as String == "original"
        }["rico:identifier"] as String? ?: error("Identifier contains no property rico:identifier: $entities.")
    }

    fun extractTypedCreationRelationAgent(
        creationRelations: List<JsonObject>,
        input: Map<String, JsonObject>,
        creationRelationTypeParam: String,
        agentTypeParam: String
    ): List<CreatorNameContainer> {
        return creationRelations.mapNotNull { creationRelation ->
            val creationRelationType = creationRelation["rico:type"] as String
            if (creationRelationTypeParam != creationRelationType) return@mapNotNull null
            val name = creationRelation["rico:name"] as String
            val agent = getEntitiesFromIds("rico:creationRelationHasTarget", creationRelation, input).first()
            val agentType = agent["@type"] as String
            if (agentType != agentTypeParam) return@mapNotNull null
            val agentName = extractLanguageContainer(agent["@id"] as String, agent["rico:name"]).first()
            return@mapNotNull CreatorNameContainer(agentName, name)
        }
    }

    fun getEntitiesFromIds(item: String, record: JsonObject, input: Map<String, JsonObject>): List<JsonObject> {
        return record[item].let {
            if (it != null) {
                when (it) {
                    is String -> listOf(input[it] as JsonObject)
                    is JsonObject -> listOf(input[it["@id"]] as JsonObject)
                    is JsonArray<*> ->
                        it.mapChildren { child ->
                            input[child["@id"]] }.filterNotNull()
                    else -> {
                        log.error("Could not parse $it to retrieve entities from input by id.")
                        emptyList()
                    }
                }
            } else emptyList()
        }
    }

    fun listOfStrings(value: Any?): List<String> {
        return when (value) {
            is String -> listOf(value)
            is List<*> ->
                if (value[0] is String)
                    value as List<String>
                else
                    emptyList()
            else ->
                emptyList()
        }
    }
}