Commit ee591026 authored by Jonas Waeber's avatar Jonas Waeber
Browse files

merge master

parents c073747c 7c61d02e
Pipeline #13563 passed with stages
in 4 minutes and 55 seconds
......@@ -61,19 +61,15 @@ dependencies {
testRuntimeOnly 'org.junit.jupiter:junit-jupiter-engine:5.6.0'
}
compileKotlin {
kotlinOptions.jvmTarget = "1.8"
}
compileTestKotlin {
tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile).configureEach {
kotlinOptions {
kotlinOptions.jvmTarget = "1.8"
}
}
test {
useJUnitPlatform()
testLogging {
events "passed", "skipped", "failed"
}
configurations {
integrationTestImplementation.extendsFrom implementation
integrationTestRuntimeOnly.extendsFrom runtimeOnly
}
sourceSets {
......@@ -83,8 +79,33 @@ sourceSets {
test.kotlin.srcDirs += 'src/test'
test.resources.srcDirs = [ "src/test/resources" ]
test.resources.includes = [ "**/*.yml", "**/*.xml", "**/*.csv"]
integrationTest {
kotlin.srcDir("src/integrationTest/kotlin")
resources.srcDir("src/integrationTest/resources")
compileClasspath += sourceSets.main.output + configurations["testRuntimeClasspath"]
runtimeClasspath += output + compileClasspath + sourceSets["test"].runtimeClasspath
}
}
test {
useJUnitPlatform()
testLogging {
events "passed", "skipped", "failed"
}
}
task integrationTest(type: Test) {
useJUnitPlatform()
testLogging {
events "passed", "skipped", "failed"
}
testClassesDirs = sourceSets.integrationTest.output.classesDirs
classpath = sourceSets.integrationTest.runtimeClasspath
shouldRunAfter test
}
plugins.withType(DistributionPlugin) {
distTar {
archiveFileName = 'app.tar'
......
......@@ -17,6 +17,10 @@
*/
package org.memobase
import java.io.File
import java.time.Duration
import java.util.Properties
import java.util.stream.Stream
import org.apache.kafka.clients.admin.AdminClient
import org.apache.kafka.clients.admin.AdminClientConfig
import org.apache.kafka.clients.consumer.ConsumerConfig
......@@ -35,10 +39,6 @@ import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.MethodSource
import org.memobase.testing.EmbeddedKafkaExtension
import org.memobase.testing.EmbeddedSftpServer
import java.io.File
import java.time.Duration
import java.util.Properties
import java.util.stream.Stream
@ExtendWith(EmbeddedKafkaExtension::class)
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
......@@ -69,14 +69,14 @@ class ServiceTest {
init {
val producerProps = Properties()
producerProps.setProperty(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:12345")
producerProps.setProperty(ProducerConfig.CLIENT_ID_CONFIG, "test-group-1")
producerProps.setProperty(ProducerConfig.CLIENT_ID_CONFIG, "input-producer")
producerProps.setProperty(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, StringSerializer::class.qualifiedName)
producerProps.setProperty(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, StringSerializer::class.qualifiedName)
inputProducer = KafkaProducer(producerProps)
val consumerProps = Properties()
consumerProps.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:12345")
consumerProps.setProperty(ConsumerConfig.CLIENT_ID_CONFIG, "test-group-1")
consumerProps.setProperty(ConsumerConfig.CLIENT_ID_CONFIG, "report-consumer")
consumerProps.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "test-group-1")
consumerProps.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer::class.qualifiedName)
consumerProps.setProperty(
......@@ -85,7 +85,7 @@ class ServiceTest {
)
consumerProps.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest")
reportConsumer = KafkaConsumer(consumerProps)
reportConsumer.subscribe(listOf("test-ingest-out"))
reportConsumer.subscribe(listOf("test-ingest-out-reporting"))
}
// TODO: Test works with local docker
......@@ -93,16 +93,18 @@ class ServiceTest {
@ParameterizedTest
@MethodSource("kafkaTests")
fun testRun(params: TestParams) {
assert(true)
/*
for (inputFileName in params.inputFileNames) {
val inputFile = File("src/integrationTest/resources/${params.count}/$inputFileName")
val id = inputFile.nameWithoutExtension
inputProducer.send(
ProducerRecord(
"test-ingest-in",
"https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69",
File("src/test/resources/integration/${params.count}/input.nt").readText()
"https://memobase.ch/record/$id",
inputFile.readText()
)
)
val expectedRecordCount = params.expectedIngestReports.size + 1
}
val expectedRecordCount = params.expectedIngestReports.size
val service = Service("test${params.count}.yml")
var inputMessages = service.consumer.fetchRecords()
......@@ -123,26 +125,27 @@ class ServiceTest {
}
assertThat(totalConsumerRecords)
.size().isEqualTo(3)
*/
.size().isEqualTo(expectedRecordCount)
}
private fun kafkaTests() = Stream.of(
TestParams(
1,
listOf(
"AFZ-IB_Becker_Audiovisuals_63.nt",
"AFZ-IB_Becker_Audiovisuals_69.nt"
),
listOf(
Report(
id = "MEI_49884",
id = "AFZ-IB_Becker_Audiovisuals_63",
status = "SUCCESS",
message = "Ingested resource MEI_49884."
message = "Ingested resource AFZ-IB_Becker_Audiovisuals_63."
),
Report(
id = "BAZ-MEI_49884-0",
id = "AFZ-IB_Becker_Audiovisuals_69",
status = "SUCCESS",
message = "Ingested resource BAZ-MEI_49884-0."
message = "Ingested resource AFZ-IB_Becker_Audiovisuals_69."
)
)
)
)
......
......@@ -20,5 +20,6 @@ package org.memobase
data class TestParams(
val count: Int,
val inputFileNames: List<String>,
val expectedIngestReports: List<Report>
)
_:Bd9c8e7b4a0c5c268362ac824fcfea65c <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Person> .
_:Bd9c8e7b4a0c5c268362ac824fcfea65c <https://www.ica.org/standards/RiC/ontology#agentIsTargetOfCreationRelation> _:B771bcc5fac4eadc746ae755fdd8d4250 .
_:B0e6a3edcff12b53509f1ce1a0d3150ce <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#CarrierType> .
_:B0e6a3edcff12b53509f1ce1a0d3150ce <https://www.ica.org/standards/RiC/ontology#name> "U-Matic"@de .
_:B3412e1c4c5370642c588bb76aae57107 <https://www.ica.org/standards/RiC/ontology#type> "contributor" .
_:B3412e1c4c5370642c588bb76aae57107 <https://www.ica.org/standards/RiC/ontology#name> "Regie" .
_:B3412e1c4c5370642c588bb76aae57107 <https://www.ica.org/standards/RiC/ontology#creationRelationHasSource> <https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> .
_:B3412e1c4c5370642c588bb76aae57107 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#CreationRelation> .
_:B3412e1c4c5370642c588bb76aae57107 <https://www.ica.org/standards/RiC/ontology#creationRelationHasTarget> _:B72f1d7b60b8f56432c89af2f28ba73a5 .
_:Bea581319X2Dfed1X2D4a4cX2Db77aX2D8b574005427e <https://www.ica.org/standards/RiC/ontology#name> "italienisch" .
_:Bea581319X2Dfed1X2D4a4cX2Db77aX2D8b574005427e <https://www.ica.org/standards/RiC/ontology#type> "content" .
_:Bea581319X2Dfed1X2D4a4cX2Db77aX2D8b574005427e <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Language> .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#scopeAndContent> "Institutionelle Archive und Bestände / Becker Audiovisuals / 2. Tonbildschauen und Multimediaschauen (1968-1997) / 2.2. Bewerbung und Präsentation von Branchen, Institutionen und Produkten / 2.2.2. Firmen und Produkte / 2.2.2.31. Schweizerische Käseunion (1988)"@de .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#isPartOf> <https://memobase.ch/recordSet/BECKER> .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#hasInstantiation> <https://memobase.ch/instantiation/physical/AFZ-IB_Becker_Audiovisuals_69-0> .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#hasSubject> _:B68e6eea7X2D8052X2D4066X2D89a2X2D5d6de7ffb376 .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#hasSubject> _:B810336cfX2Def41X2D4bb1X2Db343X2Dfb8cedfefa11 .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <http://rdaregistry.info/Elements/u/P60441> _:B4e729776cdf4111a444b1b87573ad9af .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#identifiedBy> _:B2be7b778513ddae9a08e689285124ff9 .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#regulatedBy> _:B31be9c84ff352e2fd8760736526ed271 .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#hasInstantiation> <https://memobase.ch/instantiation/digital/AFZ-IB_Becker_Audiovisuals_69-1> .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#type> "Tonbildschau" .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#title> "Die Schweizerische Käseunion AG heisst Sie willkommen in der Swisspack (Film)"@de .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <http://rdaregistry.info/Elements/u/P60451> <https://memobase.ch/institution/Memoriav> .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#hasLanguage> _:Bea581319X2Dfed1X2D4a4cX2Db77aX2D8b574005427e .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#hasSubject> _:B44e300d0X2D255cX2D46e0X2D9416X2Df77d17517eb0 .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Record> .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <http://purl.org/dc/terms/spatial> _:Be19d6f16X2D6b72X2D4cc7X2Dbbc1X2Df229859945f1 .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <http://www.ebu.ch/metadata/ontologies/ebucore/ebucore#hasGenre> _:Becdc1b39X2D102bX2D47dbX2DbdbcX2D406e95f6babd .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#recordResourceOrInstantiationIsSourceOfCreationRelation> _:B5813d2eaad318ca44eb6c808c09d160d .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#hasLanguage> _:Bcb657511X2Db019X2D4b27X2D9f6fX2D0ce00e479bec .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#identifiedBy> _:Bd073dbb0a5b647119528aa7c877142cd .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#recordResourceOrInstantiationIsSourceOfCreationRelation> _:B771bcc5fac4eadc746ae755fdd8d4250 .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <http://purl.org/dc/terms/created> _:Bdf3dfa2af995af00caa8efece83a92cb .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#hasSubject> _:Bbd0dfe5fX2D746dX2D4261X2Db151X2Da98297b648d6 .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#hasTitle> _:B89601cc6c01f7f2f8b327de6357ed403 .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#descriptiveNote> "Wortloser, mit Volksmusik untermalter Industriefilm über die Produktion und Verpackung von Käse durch die Firma Swisspack. Der Film richtet sich an ein internationales Publikum: Im Vorspann werden neben dem deutschen folgende Titel gezeigt: französisch: \"L'Union suisse du commerce de fromage SA vous Souhaite la bienvenue à la Swisspack\", italienisch: \"L'Unione svizzera per il commercio del formaggio SA Vi augura il benvenuto alla Swisspack\", englisch:\"The Swiss Cheese Union Inc. welcomes you to the Swisspack\", spanisch: \"La Unione de los quesos suizos SA augura la vienvenida a la Swisspack\". Überliefert ist: 1 Videokopie (U-Matic)"@de .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#hasSubject> _:Bc83d7a6aX2D609eX2D45ecX2D99dbX2D9a4407b94226 .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#recordResourceOrInstantiationIsSourceOfCreationRelation> _:B3412e1c4c5370642c588bb76aae57107 .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#hasSubject> _:B696014c2X2Dcf3cX2D4d2aX2Dbb13X2D6938339faf3e .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#hasLanguage> _:Bb6135bcdX2D4df0X2D4f28X2D8979X2D5d7f059e19a8 .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#heldBy> <https://memobase.ch/institution/AFZ> .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <http://www.ebu.ch/metadata/ontologies/ebucore/ebucore#hasGenre> _:Bf3aed52fX2D5cb4X2D4c3bX2D8e4eX2D8791378616ed .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#hasLanguage> _:Bd232c5b9X2D3a6fX2D4388X2Daa66X2D712b5759cfed .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <http://schema.org/sameAs> "http://onlinearchives.ethz.ch/load.aspx?guid=a50058d7-9eec-462c-9669-6abb9f26117c" .
_:B696014c2X2Dcf3cX2D4d2aX2Dbb13X2D6938339faf3e <http://www.w3.org/2004/02/skos/core#editorialNote> "Ursprungsfeld: Sachthemen" .
_:B696014c2X2Dcf3cX2D4d2aX2Dbb13X2D6938339faf3e <http://www.w3.org/2004/02/skos/core#prefLabel> "Käseverpackung" .
_:B696014c2X2Dcf3cX2D4d2aX2Dbb13X2D6938339faf3e <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2004/02/skos/core#Concept> .
_:Bd073dbb0a5b647119528aa7c877142cd <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Identifier> .
_:Bd073dbb0a5b647119528aa7c877142cd <https://www.ica.org/standards/RiC/ontology#type> "main" .
_:Bd073dbb0a5b647119528aa7c877142cd <https://www.ica.org/standards/RiC/ontology#identifier> "https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69" .
_:Bc83d7a6aX2D609eX2D45ecX2D99dbX2D9a4407b94226 <https://www.ica.org/standards/RiC/ontology#name> "Swisspack" .
_:Bc83d7a6aX2D609eX2D45ecX2D99dbX2D9a4407b94226 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Agent> .
_:B4e729776cdf4111a444b1b87573ad9af <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#CorporateBody> .
_:B4e729776cdf4111a444b1b87573ad9af <https://www.ica.org/standards/RiC/ontology#name> "Becker Audio-Visuals (BAV)"@de .
_:Ba7bdc4cf2bed30e4524d2dc7f7b56fce <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Identifier> .
_:Ba7bdc4cf2bed30e4524d2dc7f7b56fce <https://www.ica.org/standards/RiC/ontology#type> "main" .
_:Ba7bdc4cf2bed30e4524d2dc7f7b56fce <https://www.ica.org/standards/RiC/ontology#identifier> "https://memobase.ch/instantiation/physical/AFZ-IB_Becker_Audiovisuals_69-0" .
_:B5813d2eaad318ca44eb6c808c09d160d <https://www.ica.org/standards/RiC/ontology#type> "creator" .
_:B5813d2eaad318ca44eb6c808c09d160d <https://www.ica.org/standards/RiC/ontology#name> "Autor" .
_:B5813d2eaad318ca44eb6c808c09d160d <https://www.ica.org/standards/RiC/ontology#creationRelationHasSource> <https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> .
_:B5813d2eaad318ca44eb6c808c09d160d <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#CreationRelation> .
_:B5813d2eaad318ca44eb6c808c09d160d <https://www.ica.org/standards/RiC/ontology#creationRelationHasTarget> _:B3a6874626b84317c29544aac7624a8b2 .
_:Bd232c5b9X2D3a6fX2D4388X2Daa66X2D712b5759cfed <https://www.ica.org/standards/RiC/ontology#name> "englisch" .
_:Bd232c5b9X2D3a6fX2D4388X2Daa66X2D712b5759cfed <https://www.ica.org/standards/RiC/ontology#type> "content" .
_:Bd232c5b9X2D3a6fX2D4388X2Daa66X2D712b5759cfed <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Language> .
_:B44e300d0X2D255cX2D46e0X2D9416X2Df77d17517eb0 <http://www.w3.org/2004/02/skos/core#editorialNote> "Ursprungsfeld: Sachthemen" .
_:B44e300d0X2D255cX2D46e0X2D9416X2Df77d17517eb0 <http://www.w3.org/2004/02/skos/core#prefLabel> "Käseherstellung" .
_:B44e300d0X2D255cX2D46e0X2D9416X2Df77d17517eb0 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2004/02/skos/core#Concept> .
_:B89601cc6c01f7f2f8b327de6357ed403 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Title> .
_:B89601cc6c01f7f2f8b327de6357ed403 <https://www.ica.org/standards/RiC/ontology#type> "main" .
_:B89601cc6c01f7f2f8b327de6357ed403 <https://www.ica.org/standards/RiC/ontology#title> "Die Schweizerische Käseunion AG heisst Sie willkommen in der Swisspack (Film)"@de .
_:Ba8028588863644dfc91b371fa2b2be95 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Identifier> .
_:Ba8028588863644dfc91b371fa2b2be95 <https://www.ica.org/standards/RiC/ontology#type> "main" .
_:Ba8028588863644dfc91b371fa2b2be95 <https://www.ica.org/standards/RiC/ontology#identifier> "https://memobase.ch/instantiation/digital/AFZ-IB_Becker_Audiovisuals_69-1" .
_:B31be9c84ff352e2fd8760736526ed271 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Rule> .
_:B31be9c84ff352e2fd8760736526ed271 <https://www.ica.org/standards/RiC/ontology#type> "holder" .
_:B31be9c84ff352e2fd8760736526ed271 <https://www.ica.org/standards/RiC/ontology#name> "Archiv für Zeitgeschichte (ETH Zürich)" .
_:B31be9c84ff352e2fd8760736526ed271 <https://www.ica.org/standards/RiC/ontology#regulates> <https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> .
_:B68e6eea7X2D8052X2D4066X2D89a2X2D5d6de7ffb376 <https://www.ica.org/standards/RiC/ontology#name> "Käseunion" .
_:B68e6eea7X2D8052X2D4066X2D89a2X2D5d6de7ffb376 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Agent> .
_:B810336cfX2Def41X2D4bb1X2Db343X2Dfb8cedfefa11 <http://www.w3.org/2004/02/skos/core#editorialNote> "Ursprungsfeld: Sachthemen" .
_:B810336cfX2Def41X2D4bb1X2Db343X2Dfb8cedfefa11 <http://www.w3.org/2004/02/skos/core#prefLabel> "Käselagerung" .
_:B810336cfX2Def41X2D4bb1X2Db343X2Dfb8cedfefa11 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2004/02/skos/core#Concept> .
_:Bcb657511X2Db019X2D4b27X2D9f6fX2D0ce00e479bec <https://www.ica.org/standards/RiC/ontology#name> "spanisch" .
_:Bcb657511X2Db019X2D4b27X2D9f6fX2D0ce00e479bec <https://www.ica.org/standards/RiC/ontology#type> "content" .
_:Bcb657511X2Db019X2D4b27X2D9f6fX2D0ce00e479bec <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Language> .
_:Be19d6f16X2D6b72X2D4cc7X2Dbbc1X2Df229859945f1 <https://www.ica.org/standards/RiC/ontology#name> "Schweiz" .
_:Be19d6f16X2D6b72X2D4cc7X2Dbbc1X2Df229859945f1 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Place> .
_:Bb6135bcdX2D4df0X2D4f28X2D8979X2D5d7f059e19a8 <https://www.ica.org/standards/RiC/ontology#name> "deutsch" .
_:Bb6135bcdX2D4df0X2D4f28X2D8979X2D5d7f059e19a8 <https://www.ica.org/standards/RiC/ontology#type> "content" .
_:Bb6135bcdX2D4df0X2D4f28X2D8979X2D5d7f059e19a8 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Language> .
_:B3a6874626b84317c29544aac7624a8b2 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Person> .
_:B3a6874626b84317c29544aac7624a8b2 <https://www.ica.org/standards/RiC/ontology#agentIsTargetOfCreationRelation> _:B5813d2eaad318ca44eb6c808c09d160d .
_:B3a6874626b84317c29544aac7624a8b2 <https://www.ica.org/standards/RiC/ontology#name> "Egon Becker"@de .
_:Becdc1b39X2D102bX2D47dbX2DbdbcX2D406e95f6babd <http://www.w3.org/2004/02/skos/core#editorialNote> "Ursprungsfeld: Genre" .
_:Becdc1b39X2D102bX2D47dbX2DbdbcX2D406e95f6babd <http://www.w3.org/2004/02/skos/core#prefLabel> "Werbung" .
_:Becdc1b39X2D102bX2D47dbX2DbdbcX2D406e95f6babd <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2004/02/skos/core#Concept> .
_:Bf3aed52fX2D5cb4X2D4c3bX2D8e4eX2D8791378616ed <http://www.w3.org/2004/02/skos/core#editorialNote> "Ursprungsfeld: Genre" .
_:Bf3aed52fX2D5cb4X2D4c3bX2D8e4eX2D8791378616ed <http://www.w3.org/2004/02/skos/core#prefLabel> "Image" .
_:Bf3aed52fX2D5cb4X2D4c3bX2D8e4eX2D8791378616ed <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2004/02/skos/core#Concept> .
_:B437ac819636a137448a4457d2016615b <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Rule> .
_:B437ac819636a137448a4457d2016615b <https://www.ica.org/standards/RiC/ontology#type> "usage" .
_:B437ac819636a137448a4457d2016615b <https://www.ica.org/standards/RiC/ontology#name> "In Copyright (InC)" .
_:B437ac819636a137448a4457d2016615b <http://schema.org/sameAs> "http://rightsstatements.org/vocab/InC/1.0/" .
_:B437ac819636a137448a4457d2016615b <https://www.ica.org/standards/RiC/ontology#regulates> <https://memobase.ch/instantiation/digital/AFZ-IB_Becker_Audiovisuals_69-1> .
_:B9e13903910a356a5645cbee68fd40831 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Rule> .
_:B9e13903910a356a5645cbee68fd40831 <https://www.ica.org/standards/RiC/ontology#type> "access" .
_:B9e13903910a356a5645cbee68fd40831 <https://www.ica.org/standards/RiC/ontology#name> "onsite" .
_:B9e13903910a356a5645cbee68fd40831 <https://www.ica.org/standards/RiC/ontology#regulates> <https://memobase.ch/instantiation/physical/AFZ-IB_Becker_Audiovisuals_69-0> .
_:Bbd0dfe5fX2D746dX2D4261X2Db151X2Da98297b648d6 <http://www.w3.org/2004/02/skos/core#editorialNote> "Ursprungsfeld: Sachthemen" .
_:Bbd0dfe5fX2D746dX2D4261X2Db151X2Da98297b648d6 <http://www.w3.org/2004/02/skos/core#prefLabel> "Käseportionierung" .
_:Bbd0dfe5fX2D746dX2D4261X2Db151X2Da98297b648d6 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2004/02/skos/core#Concept> .
_:B2be7b778513ddae9a08e689285124ff9 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Identifier> .
_:B2be7b778513ddae9a08e689285124ff9 <https://www.ica.org/standards/RiC/ontology#type> "original" .
_:B2be7b778513ddae9a08e689285124ff9 <https://www.ica.org/standards/RiC/ontology#identifier> "IB Becker Audiovisuals / 69" .
_:B1d003c286b7072a7453f0a5cd22701a0 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Identifier> .
_:B1d003c286b7072a7453f0a5cd22701a0 <https://www.ica.org/standards/RiC/ontology#type> "callNumber" .
_:B1d003c286b7072a7453f0a5cd22701a0 <https://www.ica.org/standards/RiC/ontology#identifier> "IB Becker Audiovisuals / 69" .
_:B0d88137eb22e7e5ed6f58c6922f9131d <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Rule> .
_:B0d88137eb22e7e5ed6f58c6922f9131d <https://www.ica.org/standards/RiC/ontology#type> "access" .
_:B0d88137eb22e7e5ed6f58c6922f9131d <https://www.ica.org/standards/RiC/ontology#name> "public" .
_:B0d88137eb22e7e5ed6f58c6922f9131d <https://www.ica.org/standards/RiC/ontology#regulates> <https://memobase.ch/instantiation/digital/AFZ-IB_Becker_Audiovisuals_69-1> .
_:Bdf3dfa2af995af00caa8efece83a92cb <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#DateRange> .
_:Bdf3dfa2af995af00caa8efece83a92cb <https://www.ica.org/standards/RiC/ontology#normalizedDateValue> "1988" .
_:B72f1d7b60b8f56432c89af2f28ba73a5 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Person> .
_:B72f1d7b60b8f56432c89af2f28ba73a5 <https://www.ica.org/standards/RiC/ontology#agentIsTargetOfCreationRelation> _:B3412e1c4c5370642c588bb76aae57107 .
_:B72f1d7b60b8f56432c89af2f28ba73a5 <https://www.ica.org/standards/RiC/ontology#name> "Egon Becker"@de .
_:B771bcc5fac4eadc746ae755fdd8d4250 <https://www.ica.org/standards/RiC/ontology#type> "contributor" .
_:B771bcc5fac4eadc746ae755fdd8d4250 <https://www.ica.org/standards/RiC/ontology#name> "Kamera" .
_:B771bcc5fac4eadc746ae755fdd8d4250 <https://www.ica.org/standards/RiC/ontology#creationRelationHasSource> <https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> .
_:B771bcc5fac4eadc746ae755fdd8d4250 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#CreationRelation> .
_:B771bcc5fac4eadc746ae755fdd8d4250 <https://www.ica.org/standards/RiC/ontology#creationRelationHasTarget> _:Bd9c8e7b4a0c5c268362ac824fcfea65c .
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Fedora Ingest Service
~ Copyright (C) 2020 Project Memobase
~
~ This program is free software: you can redistribute it and/or modify
~ it under the terms of the GNU Affero General Public License as published by
~ the Free Software Foundation, either version 3 of the License, or
~ (at your option) any later version.
~
~ This program is distributed in the hope that it will be useful,
~ but WITHOUT ANY WARRANTY; without even the implied warranty of
~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
~ GNU Affero General Public License for more details.
~
~ You should have received a copy of the GNU Affero General Public License
~ along with this program. If not, see <https://www.gnu.org/licenses/>.
-->
<Configuration status="info" name="skeleton-app" packages="">
<Appenders>
<Console name="STDOUT" target="SYSTEM_OUT">
<PatternLayout pattern="[%-5level] [%c{1}] %m%n"/>
</Console>
</Appenders>
<Loggers>
<Root level="error">
<AppenderRef ref="STDOUT"/>
</Root>
</Loggers>
</Configuration>
\ No newline at end of file
......@@ -8,7 +8,7 @@ app:
fedora:
username: fedoraAdmin
password: fedoraAdmin
internalBaseUrl: http://localhost:8081/fcrepo/rest/
internalBaseUrl: http://localhost:8080/fcrepo/rest/
externalBaseUrl: https://memobase.ch/
kafka:
consumer:
......@@ -21,4 +21,4 @@ kafka:
topic:
in: test-ingest-in
out: test-ingest-out
process: test-ingest-report
\ No newline at end of file
process: test-ingest-reporting
\ No newline at end of file
......@@ -73,38 +73,32 @@ class Ingester(
val recordOutput = StringWriter()
val recordPair = rdfHandler.getRecord()
recordPair.second.write(recordOutput, "NTRIPLES")
try {
// FIXME: correct for ingest:
// 1 binaries
// 2 instantiations
// 3 records
val data = recordOutput.toString()
log.info("Ingesting record ${recordPair.first}.")
fedoraClient.createOrUpdateRdfResource(URI(recordPair.first), data, RdfContentTypes.NTRIPLES)
log.info("Ingested record ${recordPair.first}.")
producer.sendReport(
Report(recordPair.first, ReportStatus.success, "Ingested record in fedora.")
)
} catch (ex: FcrepoOperationFailedException) {
log.error("Ingestion of record ${recordPair.first} failed: ${ex.localizedMessage}.")
producer.sendReport(
Report(recordPair.first, ReportStatus.failure, ex.localizedMessage)
)
}
// FIXME: extract into separate method
rdfHandler.getInstantiations().forEach { instantiationPair ->
val instantiationOutput = StringWriter()
instantiationPair.second.write(instantiationOutput, "NTRIPLES")
try {
val data = instantiationOutput.toString()
fedoraClient.createOrUpdateRdfResource(URI(instantiationPair.first), data, RdfContentTypes.NTRIPLES)
val instantiationData = instantiationOutput.toString()
log.info("Ingesting instantiation ${instantiationPair.first}.")
fedoraClient.createOrUpdateRdfResource(URI(instantiationPair.first), instantiationData, RdfContentTypes.NTRIPLES)
log.info("Ingested instantiation ${instantiationPair.first}.")
producer.sendReport(
Report(instantiationPair.first, ReportStatus.success, "Ingested an instantiation in fedora.")
)
} catch (ex: FcrepoOperationFailedException) {
log.error("Ingestion of instantiation ${instantiationPair.first} failed: ${ex.localizedMessage}.")
producer.sendReport(
Report(instantiationPair.first, ReportStatus.failure, ex.localizedMessage)
)
throw ex
}
}
// FIXME: extract into separate method
rdfHandler.getSftpLocators().forEach {
val digitalInstantiationUrl = it.first
it.second.let { path ->
......@@ -117,14 +111,9 @@ class Ingester(
}.second
log.info("Ingesting binary $binaryUri with mime type $mimeType.")
fedoraClient.createOrUpdateBinaryResource(URI(binaryUri), stream, mimeType)
producer.sendReport(
Report(binaryUri, ReportStatus.success, "Ingested a binary in fedora.")
)
} catch (ex: FcrepoOperationFailedException) {
log.error("Ingestion of binary $binaryUri failed: ${ex.localizedMessage}.")
producer.sendReport(
Report(binaryUri, ReportStatus.failure, ex.localizedMessage)
)
throw ex
}
}
}
......@@ -149,6 +138,7 @@ class Ingester(
message = ReportMessages.ingestedRecord(record.key())
)
} catch (e: FcrepoOperationFailedException) {
log.error("Ingestion of record ${record.key()} failed: ${e.localizedMessage}.")
Report(
id = record.key(),
status = ReportStatus.failure,
......
......@@ -17,10 +17,10 @@
*/
package org.memobase
import org.apache.kafka.clients.producer.KafkaProducer
import org.apache.kafka.clients.producer.ProducerRecord
import java.io.Closeable
import java.util.Properties
import org.apache.kafka.clients.producer.KafkaProducer
import org.apache.kafka.clients.producer.ProducerRecord
class Producer(props: Properties, topic: String) : Closeable {
private val instance = KafkaProducer<String, String>(props)
......
package org.memobase
import java.io.ByteArrayInputStream
import org.apache.jena.rdf.model.Model
import org.apache.jena.rdf.model.ModelFactory
import org.apache.jena.rdf.model.Statement
......@@ -9,7 +10,6 @@ import org.apache.log4j.LogManager
import org.memobase.rdf.EBUCORE
import org.memobase.rdf.RDF
import org.memobase.rdf.RICO
import java.io.ByteArrayInputStream
class RdfHandler(data: String) {
private val log = LogManager.getLogger("RdfHandler")
......
......@@ -19,18 +19,17 @@
package org.memobase
// import org.bouncycastle.cms.RecipientId.password
import java.util.Properties
import org.apache.logging.log4j.LogManager
import org.apache.logging.log4j.Logger
import org.memobase.fedora.FedoraClient
import org.memobase.fedora.FedoraClientImpl
import org.memobase.settings.SettingsLoader
import org.memobase.sftp.SftpClient
import java.util.Properties
class Service(fileName: String = "app.yml") {
companion object {
const val JOB_REPORT_ID = "fedora-ingest"
const val FEDORA_PROPERTIES_PREFIX = "fedora"
fun createFedoraClient(appSettings: Properties): FedoraClient {
......
package org.memobase
import java.io.File
import java.io.StringWriter
import java.nio.charset.Charset
import org.apache.jena.riot.Lang
import org.apache.jena.riot.RDFDataMgr
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.TestInstance
import java.io.File
import java.io.StringWriter
import java.nio.charset.Charset
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class TestRdfHandler {
......@@ -18,7 +17,6 @@ class TestRdfHandler {
return File("$resourcePath/$fileName").readText(Charset.defaultCharset())
}
private val regex = Regex("(_:B[A-Za-z0-9]+)")
private fun sort(input: List<String>): String {
......@@ -112,10 +110,8 @@ class TestRdfHandler {
assertThat(rdfHandler.getMimeType().first().second).isEqualTo("video/mpeg")
}
private val uris2 = listOf(
"https://memobase.ch/instantiation/digital/AFZ-IB_Becker_Audiovisuals_63-1",
"https://memobase.ch/instantiation/digital/AFZ-IB_Becker_Audiovisuals_63-2"
)
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment