Commit 49f6066d authored by Jonas Waeber's avatar Jonas Waeber

Update dependencies.

parent 18c9ec92
Pipeline #18489 passed with stages
in 4 minutes and 2 seconds
......@@ -35,8 +35,8 @@ dependencies {
//compile group: 'org.elasticsearch.client', name: 'elasticsearch-rest-high-level-client', version: '7.1.0'
implementation 'ch.memobase:memobase-kafka-utils:0.1.2'
implementation 'org.memobase:memobase-service-utilities:1.12.3'
implementation 'ch.memobase:normalization-service-configuration:0.1.6'
implementation 'org.memobase:memobase-service-utilities:2.0.2'
implementation 'ch.memobase:normalization-service-configuration:0.2.0'
// Logging Framework
implementation "org.apache.logging.log4j:log4j-api:${log4jV}"
......
......@@ -24,6 +24,11 @@ import ch.memobase.kafka.utils.ConfigJoiner
import ch.memobase.kafka.utils.models.ImportService
import ch.memobase.kafka.utils.models.JoinedValues
import ch.memobase.rdf.MemobaseModel
import ch.memobase.reporting.Report
import ch.memobase.reporting.ReportStatus
import ch.memobase.settings.HeaderExtractionTransformSupplier
import ch.memobase.settings.HeaderMetadata
import ch.memobase.settings.SettingsLoader
import org.apache.jena.rdf.model.Model
import org.apache.jena.riot.Lang
import org.apache.jena.riot.RDFDataMgr
......@@ -33,9 +38,6 @@ import org.apache.kafka.streams.KeyValue
import org.apache.kafka.streams.StreamsBuilder
import org.apache.kafka.streams.kstream.Predicate
import org.apache.logging.log4j.LogManager
import org.memobase.settings.SettingsLoader
import settings.HeaderExtractionTransformSupplier
import settings.HeaderMetadata
import java.io.ByteArrayInputStream
import java.io.StringWriter
......@@ -76,7 +78,15 @@ class KafkaTopology(
)
filterInvalidLocalTransforms[0]
.mapValues { readOnlyKey, value -> Report(readOnlyKey, ReportStatus.failure, value.first.right.errorMessage).toJson() }
.mapValues { readOnlyKey, value ->
Report(
readOnlyKey,
ReportStatus.fatal,
value.first.right.errorMessage,
Service.step
).toJson()
}
.to(reportTopic)
val parsedModelStream = filterInvalidLocalTransforms[1]
......@@ -90,8 +100,9 @@ class KafkaTopology(
.mapValues { key, value ->
Report(
key,
ReportStatus.failure,
"Could not parse message. Found invalid input data (RiotException): $value"
ReportStatus.fatal,
"Could not parse message. Found invalid input data (RiotException): $value",
Service.step
).toJson()
}
.to(reportTopic)
......@@ -132,9 +143,10 @@ class KafkaTopology(
return Pair(
Report(
key,
if (results.isEmpty()) ReportStatus.success else ReportStatus.failure,
if (results.isEmpty()) ReportStatus.success else ReportStatus.fatal,
if (results.isEmpty()) "Normalization was successful." else results.reduce { acc, s -> acc + "\n" + s }
.trim()
.trim(),
Service.step
),
input.model
)
......
......@@ -2,7 +2,7 @@ package org.memobase
import ch.memobase.configs.LocalTransformsLoader
import ch.memobase.rdf.MemobaseModel
import settings.HeaderMetadata
import ch.memobase.settings.HeaderMetadata
data class PreparedInput(
val model: MemobaseModel,
......
/*
* Copyright (C) 2020 Memoriav
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.memobase
import com.beust.klaxon.Klaxon
import java.time.LocalDateTime
data class Report(
val id: String,
val status: String,
val message: String,
val step: String = "normalization-service",
val timestamp: String = LocalDateTime.now().toString()
) {
fun toJson(): String {
return Klaxon().toJsonString(this)
}
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (javaClass != other?.javaClass) return false
other as Report
if (id != other.id) return false
if (status != other.status) return false
if (message != other.message) return false
if (step != other.step) return false
return true
}
override fun hashCode(): Int {
var result = id.hashCode()
result = 31 * result + status.hashCode()
result = 31 * result + message.hashCode()
result = 31 * result + step.hashCode()
return result
}
}
/*
* mapper-service
* Copyright (C) 2020 Memoriav
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.memobase
object ReportStatus {
const val success = "SUCCESS"
const val failure = "FAILURE"
}
......@@ -18,10 +18,12 @@ package org.memobase
import org.apache.kafka.streams.KafkaStreams
import org.apache.logging.log4j.LogManager
import org.memobase.settings.SettingsLoader
import kotlin.system.exitProcess
import ch.memobase.settings.SettingsLoader
class Service(file: String = "app.yml") {
companion object {
const val step = "normalization-service"
}
private val log = LogManager.getLogger("NormalizationService")
val settings = SettingsLoader(
......
......@@ -18,6 +18,8 @@
package org.memobase
import ch.memobase.rdf.MemobaseModel
import ch.memobase.rdf.NS
import ch.memobase.reporting.Report
import com.beust.klaxon.Klaxon
import org.apache.jena.riot.Lang
import org.apache.jena.riot.RDFDataMgr
......@@ -33,7 +35,6 @@ import org.junit.jupiter.api.TestInstance
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.MethodSource
import org.memobase.params.KafkaTestParams
import org.memobase.rdf.NS
import java.io.File
import java.io.FileOutputStream
import java.nio.charset.Charset
......@@ -67,7 +68,8 @@ class TestIntegration {
@MethodSource("kafkaTests")
fun `test kafka topology`(params: KafkaTestParams) {
val service = Service("test${params.count}.yml")
val testDriver = TopologyTestDriver(KafkaTopology(service.settings).prepare().build(), service.settings.kafkaStreamsSettings)
val testDriver =
TopologyTestDriver(KafkaTopology(service.settings).prepare().build(), service.settings.kafkaStreamsSettings)
val factory = ConsumerRecordFactory(StringSerializer(), StringSerializer())
val input = readFile("${params.count}/input.nt")
val output = readFile("${params.count}/output.nt").lines()
......@@ -156,7 +158,7 @@ class TestIntegration {
KafkaTestParams(
5,
"placeholder",
Report("placeholder", "SUCCESS", "Normalization was successful.")
Report("placeholder", "SUCCESS", "Normalization was successful.", Service.step)
)/*,
KafkaTestParams(
6,
......
......@@ -18,7 +18,7 @@
package org.memobase.params
import org.memobase.Report
import ch.memobase.reporting.Report
data class KafkaTestParams(
val count: Int,
......
@prefix schema: <http://schema.org/> .
@prefix internal: <http://memobase.ch/internal/> .
@prefix internal: <https://memobase.ch/internal/> .
@prefix mbrs: <https://memobase.ch/recordSet/> .
@prefix owl: <http://www.w3.org/2002/07/owl#> .
@prefix wdt: <http://www.wikidata.org/prop/direct/> .
......@@ -22,6 +22,16 @@
@prefix foaf: <http://xmlns.com/foaf/0.1/> .
@prefix dc: <http://purl.org/dc/elements/1.1/> .
_:b0 a rico:Activity ;
rico:beginningDate "2020-26-07T17:26:36+0000"^^xsd:dateTime ;
rico:endDate "2020-26-07T17:26:36+0000"^^xsd:dateTime ;
rico:performedBy [ a rico:Mechanism ;
rico:name "GenreNormalizer" ;
rico:performs _:b0
] ;
rico:resultsIn _:b1 ;
rico:type "enrichment" .
mbr:sfw-SFW_1376-2 a rico:Record ;
dcterms:abstract "Abstract: <br> Schweiz, Seedorf, UR: Moderner Landwirtschaftsbetrieb der Nonnen des Benediktinerinnen-Klosters <br> <br> Sequenzbeschrieb: <br> Seedorf, UR (Schweiz) – Schwenk AA Kloster <br> Seedorf, UR (Schweiz) – Benediktinerinnen bei Gartenarbeit <br> Seedorf, UR (Schweiz) – Nonne bei Feldarbeit auf dem Traktor, Ordensschwestern Heu wendend <br> Seedorf, UR (Schweiz) – Aufladen von Heu mit Ladewagen <br> Seedorf, UR (Schweiz) – Einfahrt Traktor mit vollem Heuwagen in Scheune <br> <br> Communiqué: <br> In Seedorf, Kanton Uri, gehören seit vielen hundert Jahren 6 Stunden Feldarbeit zum Tagesprogramm des dortigen Benediktinerinnen-Klosters. Heute verfügt es auch über die modernsten Erntemaschinen. <br> <br> Begleittext: <br> Die frommen Klosterfrauen von Seedorf im Reusstal sind bekannt dafür, dass sie mindestens 6 Stunden ihres Tagesprogramms der Garten-und Feldarbeit widmen. / Modernste Landwirtschaftsmaschinen erlauben den Benediktinerinnen sowohl sich selbst als auch eine dem Kloster angeschlossene Haushaltungsschule mit 30 jungen Töchtern reichhaltig zu ernähren. / Zur Zeit der Kreuzzüge stand hier ein Siechenhaus, das dem heiligen Lazarus geweiht war und schon damals viele hilfreiche weibliche Geister aus der ganzen Innerschweiz anzog. / Im 16. Jahrhundert übergab Papst Paul IV. das vom Kanton renovierte Kloster den Benediktinerinnen, die es seither geradezu mustergültig betreut haben. / Wenn der viele Regen das Gras nicht trocknen lässt, dann betreiben die fleissigen Nonnen das Emden in zwei Gruppen. Während die eine das Gras schneidet, fährt die andere mit der Maschine direkt hintendrein und lädt alles auf. / Das nasse Gras wird dann ins Kloster gefahren und dort in einer Spezialanlage getrocknet. Bei den Seedorfer Klosterfrauen ist es Tradition, weitherum im Lande als besonders aufgeschlossen und fortschrittlich zu gelten. <br> <br> <![CDATA[<a href=\"https://media.zem.ch/01WS/1969/Communique_1376.pdf\" target=\"_blank\">Communiqué_1376.pdf</a>]]>"@de ;
dcterms:issued [ a rico:DateRange ;
......@@ -38,33 +48,33 @@ mbr:sfw-SFW_1376-2 a rico:Record ;
rico:name "Seedorf, UR (Schweiz)"@de
] ;
schema:sameAs "https://www.recherche.bar.admin.ch/recherche/#/de/suche/archivplan/21689176" ;
ebucore:hasGenre _:b0 , _:b1 , _:b2 ;
ebucore:hasGenre _:b2 , _:b3 , _:b1 ;
ebucore:hasGenre [ a skos:Concept ;
skos:editorialNote "Ursprungsfeld: Genre" ;
skos:prefLabel "Canzone"@de
skos:prefLabel "Ballett"@de
] ;
ebucore:hasGenre [ a skos:Concept ;
skos:editorialNote "Ursprungsfeld: Genre" ;
skos:prefLabel "Ballett"@de
skos:prefLabel "Filmwochenschau"@de
] ;
ebucore:hasGenre [ a skos:Concept ;
skos:editorialNote "Ursprungsfeld: Genre" ;
skos:prefLabel "Filmwochenschau"@de
skos:prefLabel "Canzone"@de
] ;
rico:descriptiveNote "Quellenangabe für Zitate und Publikationen: <br> Schweizerisches Bundesarchiv, J2.143#1996/386#1376-1#2*, Schweizer Filmwochenschau vom 19.09.1969, Klosterfrauen treiben Agrarkultur (1376-2)"@de ;
rico:hasInstantiation <https://memobase.ch/instantiation/digital/sfw-SFW_1376-2-1> , <https://memobase.ch/instantiation/physical/sfw-SFW_1376-2-0> ;
rico:hasLanguage _:b3 ;
rico:hasLanguage _:b4 ;
rico:hasLanguage [ a rico:Language ;
rico:name "dt"@de ;
rico:type "content"
] ;
rico:hasSubject [ a skos:Concept ;
skos:editorialNote "Ursprungsfeld: Deskriptoren" ;
skos:prefLabel "ORDENSGEMEINSCHAFT"@de
skos:prefLabel "CHRISTLICHE KIRCHE"@de
] ;
rico:hasSubject [ a skos:Concept ;
skos:editorialNote "Ursprungsfeld: Deskriptoren" ;
skos:prefLabel "CHRISTLICHE KIRCHE"@de
skos:prefLabel "RELIGION"@de
] ;
rico:hasSubject [ a skos:Concept ;
skos:editorialNote "Ursprungsfeld: Deskriptoren" ;
......@@ -72,23 +82,23 @@ mbr:sfw-SFW_1376-2 a rico:Record ;
] ;
rico:hasSubject [ a skos:Concept ;
skos:editorialNote "Ursprungsfeld: Deskriptoren" ;
skos:prefLabel "RELIGION"@de
skos:prefLabel "ORDENSGEMEINSCHAFT"@de
] ;
rico:hasSubject [ a skos:Concept ;
skos:editorialNote "Ursprungsfeld: Deskriptoren" ;
skos:prefLabel "LANDWIRTSCHAFT"@de
] ;
rico:hasTitle [ a rico:Title ;
rico:title "Klosterfrauen treiben Agrarkultur (1376-2)"@de ;
rico:type "main"
rico:title "Schweizer Filmwochenschau"@de ;
rico:type "series"
] ;
rico:hasTitle [ a rico:Title ;
rico:title "Schweizer Filmwochenschau vom 19.09.1969"@de ;
rico:type "broadcast"
] ;
rico:hasTitle [ a rico:Title ;
rico:title "Schweizer Filmwochenschau"@de ;
rico:type "series"
rico:title "Klosterfrauen treiben Agrarkultur (1376-2)"@de ;
rico:type "main"
] ;
rico:heldBy mbcb:sfw ;
rico:identifiedBy [ a rico:Identifier ;
......@@ -104,7 +114,7 @@ mbr:sfw-SFW_1376-2 a rico:Record ;
rico:name "SFW"@de
] ;
rico:recordResourceOrInstantiationIsSourceOfCreationRelation
_:b4 ;
_:b5 ;
rico:regulatedBy [ a rico:Rule ;
rico:name "Cinémathèque suisse, Schweizerisches Bundesarchiv" ;
rico:regulates mbr:sfw-SFW_1376-2 ;
......@@ -113,21 +123,26 @@ mbr:sfw-SFW_1376-2 a rico:Record ;
rico:title "Klosterfrauen treiben Agrarkultur (1376-2)"@de ;
rico:type "Film" .
_:b5 a rico:Activity ;
rico:beginningDate "2020-57-20T17:57:23+0000"^^xsd:dateTime ;
rico:endDate "2020-57-20T17:57:23+0000"^^xsd:dateTime ;
rico:performedBy [ a rico:Mechanism ;
rico:name "CarrierTypeNormalizer" ;
rico:performs _:b5
] ;
rico:resultsIn _:b6 ;
rico:type "enrichment" .
_:b0 a skos:Concept ;
_:b3 a skos:Concept ;
skos:editorialNote "Ursprungsfeld: Genre" ;
skos:prefLabel "Performance"@de , "Performance"@fr , "Prestazioni"@it ;
rico:resultsFrom _:b6 .
_:b2 a skos:Concept ;
skos:editorialNote "Ursprungsfeld: Genre" ;
skos:prefLabel "Filmwochenschau"@de , "Filmwochenschau"@fr , "Filmwochenschau"@it ;
rico:resultsFrom _:b7 .
_:b7 a rico:Activity ;
rico:beginningDate "2020-26-07T17:26:36+0000"^^xsd:dateTime ;
rico:endDate "2020-26-07T17:26:36+0000"^^xsd:dateTime ;
rico:performedBy [ a rico:Mechanism ;
rico:name "GenreNormalizer" ;
rico:performs _:b7
] ;
rico:resultsIn _:b2 ;
rico:type "enrichment" .
<https://memobase.ch/instantiation/digital/sfw-SFW_1376-2-1>
a rico:Instantiation ;
ebucore:locator "<![CDATA[https://media.zem.ch/01WS/1969/SFW_1376.mp4?start=109&end=187]]>" ;
......@@ -151,17 +166,16 @@ _:b0 a skos:Concept ;
] ;
rico:type "digitalObject" .
_:b3 a rico:Language ;
schema:sameAs "http://www.wikidata.org/entity/Q188" ;
rico:name "Deutsch"@de , "allemand"@fr , "tedesco"@it ;
rico:resultsFrom _:b8 ;
rico:type "content" .
_:b1 a skos:Concept ;
skos:editorialNote "Ursprungsfeld: Genre" ;
skos:prefLabel "Musik"@de , "Musique"@fr , "Musica"@it ;
rico:resultsFrom _:b0 .
<https://memobase.ch/instantiation/physical/sfw-SFW_1376-2-0>
a rico:Instantiation ;
rdau:P60558 "SW"@de ;
ebucore:duration "78" ;
rico:hasCarrierType _:b6 ;
rico:hasCarrierType _:b8 ;
rico:hasCarrierType [ a rico:CarrierType ;
rico:name "35mm Zelluloseazetat"@de
] ;
......@@ -177,67 +191,53 @@ _:b3 a rico:Language ;
rico:instantiates mbr:sfw-SFW_1376-2 ;
rico:type "physicalObject" .
_:b9 a rico:Activity ;
rico:beginningDate "2020-57-20T17:57:23+0000"^^xsd:dateTime ;
rico:endDate "2020-57-20T17:57:23+0000"^^xsd:dateTime ;
_:b6 a rico:Activity ;
rico:beginningDate "2020-26-07T17:26:36+0000"^^xsd:dateTime ;
rico:endDate "2020-26-07T17:26:36+0000"^^xsd:dateTime ;
rico:performedBy [ a rico:Mechanism ;
rico:name "GenreNormalizer" ;
rico:performs _:b9
rico:performs _:b6
] ;
rico:resultsIn _:b2 ;
rico:resultsIn _:b3 ;
rico:type "enrichment" .
_:b1 a skos:Concept ;
skos:editorialNote "Ursprungsfeld: Genre" ;
skos:prefLabel "Filmwochenschau"@de , "Filmwochenschau"@fr , "Filmwochenschau"@it ;
rico:resultsFrom _:b10 .
_:b4 a rico:Language ;
schema:sameAs "http://www.wikidata.org/entity/Q188" ;
rico:name "Deutsch"@de , "allemand"@fr , "tedesco"@it ;
rico:resultsFrom _:b9 ;
rico:type "content" .
_:b10 a rico:Activity ;
rico:beginningDate "2020-57-20T17:57:23+0000"^^xsd:dateTime ;
rico:endDate "2020-57-20T17:57:23+0000"^^xsd:dateTime ;
rico:beginningDate "2020-26-07T17:26:36+0000"^^xsd:dateTime ;
rico:endDate "2020-26-07T17:26:36+0000"^^xsd:dateTime ;
rico:performedBy [ a rico:Mechanism ;
rico:name "GenreNormalizer" ;
rico:name "CarrierTypeNormalizer" ;
rico:performs _:b10
] ;
rico:resultsIn _:b1 ;
rico:resultsIn _:b8 ;
rico:type "enrichment" .
_:b4 a rico:CreationRelation ;
_:b5 a rico:CreationRelation ;
rico:creationRelationHasSource mbr:sfw-SFW_1376-2 ;
rico:creationRelationHasTarget [ a rico:CorporateBody ;
rico:agentIsTargetOfCreationRelation
_:b4 ;
_:b5 ;
rico:name "Schweizer Filmwochenschau (SFW)"@de
] ;
rico:name "Autor" ;
rico:type "creator" .
_:b7 a rico:Activity ;
rico:beginningDate "2020-57-20T17:57:23+0000"^^xsd:dateTime ;
rico:endDate "2020-57-20T17:57:23+0000"^^xsd:dateTime ;
rico:performedBy [ a rico:Mechanism ;
rico:name "GenreNormalizer" ;
rico:performs _:b7
] ;
rico:resultsIn _:b0 ;
rico:type "enrichment" .
_:b6 a rico:CarrierType ;
_:b8 a rico:CarrierType ;
schema:sameAs "http://www.wikidata.org/entity/Q226528" ;
rico:name "35-mm-Film"@de , "format 35 mm"@fr , "35 millimetri"@it ;
rico:resultsFrom _:b5 .
rico:resultsFrom _:b10 .
_:b8 a rico:Activity ;
rico:beginningDate "2020-57-20T17:57:23+0000"^^xsd:dateTime ;
rico:endDate "2020-57-20T17:57:23+0000"^^xsd:dateTime ;
_:b9 a rico:Activity ;
rico:beginningDate "2020-26-07T17:26:36+0000"^^xsd:dateTime ;
rico:endDate "2020-26-07T17:26:36+0000"^^xsd:dateTime ;
rico:performedBy [ a rico:Mechanism ;
rico:name "LanguagesNormalizer" ;
rico:performs _:b8
rico:performs _:b9
] ;
rico:resultsIn _:b3 ;
rico:resultsIn _:b4 ;
rico:type "enrichment" .
_:b2 a skos:Concept ;
skos:editorialNote "Ursprungsfeld: Genre" ;
skos:prefLabel "Musik"@de , "Musique"@fr , "Musica"@it ;
rico:resultsFrom _:b9 .
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment