Commit 65866f9d authored by Thomas Bernhart's avatar Thomas Bernhart
Browse files

Merge branch 'MEMO-650-create-placeholders-for-specific-resource-types-only' into 'master'

Memo 650 create placeholders for specific resource types only

See merge request !3
parents 7c61d02e 7d6b2d77
Pipeline #13711 failed with stages
in 2 minutes and 35 seconds
......@@ -32,6 +32,7 @@ ext {
}
dependencies {
def excludeSlf4J = { exclude group: 'org.slf4j' }
// JSON Parser
implementation 'com.beust:klaxon:5.2'
......@@ -40,14 +41,12 @@ dependencies {
implementation "org.apache.logging.log4j:log4j-core:${log4jV}"
implementation "org.apache.logging.log4j:log4j-slf4j-impl:${log4jV}"
implementation "org.apache.kafka:kafka-clients:${kafkaV}"
implementation "org.apache.kafka:kafka-clients:${kafkaV}", excludeSlf4J
//implementation 'org.fcrepo.client:fcrepo-java-client:0.4.0'
implementation 'org.apache.jena:apache-jena:3.14.0', excludeSlf4J
implementation 'org.apache.jena:apache-jena:3.14.0'
implementation 'org.memobase:memobase-service-utilities:1.10.0'
implementation 'org.memobase:fedora-client:0.4.1'
implementation 'org.memobase:memobase-service-utilities:1.10.0', excludeSlf4J
implementation 'org.memobase:fedora-client:0.5.0', excludeSlf4J
// KOTLIN IMPORTS
implementation 'org.jetbrains.kotlin:kotlin-stdlib-jdk8'
......
......@@ -18,6 +18,7 @@
package org.memobase
import java.io.File
import java.io.FileInputStream
import java.time.Duration
import java.util.Properties
import java.util.stream.Stream
......@@ -51,16 +52,14 @@ class ServiceTest {
AdminClient.create(mapOf(Pair(AdminClientConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:12345")))
init {
// TODO: setup embedded sftpServer with test data
/*
// setup embedded sftpServer with test data
sftpServer.createDirectories(
"/memobase/test_institution_1/test_record_set_1/"
"/memobase/AFZ/BECKER/"
)
sftpServer.putFile(
"/memobase/test_institution_1/test_record_set_1/binary.txt",
FileInputStream("src/test/resources/data/binary.txt")
"/memobase/AFZ/BECKER/binary.txt",
FileInputStream("src/integrationTest/resources/sftpData/binary.txt")
)
*/
}
private val inputProducer: KafkaProducer<String, String>
......@@ -89,7 +88,7 @@ class ServiceTest {
}
// TODO: Test works with local docker
// docker run --rm -p 8081:8080 --name=fcrepo5 fcrepo/fcrepo:5.1.0
// docker run --rm -p 8080:8080 --name=fcrepo5 fcrepo/fcrepo:5.1.0
@ParameterizedTest
@MethodSource("kafkaTests")
fun testRun(params: TestParams) {
......@@ -132,19 +131,13 @@ class ServiceTest {
TestParams(
1,
listOf(
"AFZ-IB_Becker_Audiovisuals_63.nt",
"AFZ-IB_Becker_Audiovisuals_69.nt"
"AFZ-IB_Becker_Audiovisuals_63.nt"
),
listOf(
Report(
id = "AFZ-IB_Becker_Audiovisuals_63",
status = "SUCCESS",
message = "Ingested resource AFZ-IB_Becker_Audiovisuals_63."
),
Report(
id = "AFZ-IB_Becker_Audiovisuals_69",
status = "SUCCESS",
message = "Ingested resource AFZ-IB_Becker_Audiovisuals_69."
)
)
)
......
......@@ -142,3 +142,9 @@ _:B3144591eX2D60bbX2D4e8dX2D8990X2D81bd3a77397c <http://www.w3.org/1999/02/22-rd
_:B8a908fe981878ee6c31e646a803cf412 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Person> .
_:B8a908fe981878ee6c31e646a803cf412 <https://www.ica.org/standards/RiC/ontology#agentIsTargetOfCreationRelation> _:B16fa81508225a906388e56d5fb8fbc75 .
_:B8a908fe981878ee6c31e646a803cf412 <https://www.ica.org/standards/RiC/ontology#name> "Egon Becker"@de .
<https://memobase.ch/instantiation/digital/AFZ-IB_Becker_Audiovisuals_63-2> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Instantiation> .
<https://memobase.ch/instantiation/digital/AFZ-IB_Becker_Audiovisuals_63-2> <https://www.ica.org/standards/RiC/ontology#type> "digitalObject" .
<https://memobase.ch/instantiation/digital/AFZ-IB_Becker_Audiovisuals_63-2> <http://www.ebu.ch/metadata/ontologies/ebucore/ebucore#locator> "sftp:/memobase/AFZ/BECKER/binary.txt" .
<https://memobase.ch/instantiation/digital/AFZ-IB_Becker_Audiovisuals_63-2> <http://www.ebu.ch/metadata/ontologies/ebucore/ebucore#hasMimeType> "text/plain" .
<https://memobase.ch/instantiation/digital/AFZ-IB_Becker_Audiovisuals_63-2> <https://www.ica.org/standards/RiC/ontology#instantiates> <https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_63> .
<https://memobase.ch/instantiation/digital/AFZ-IB_Becker_Audiovisuals_63-2> <https://www.ica.org/standards/RiC/ontology#isDerivedFromInstantiation> <https://memobase.ch/instantiation/physical/AFZ-IB_Becker_Audiovisuals_63-0> .
\ No newline at end of file
_:Bd9c8e7b4a0c5c268362ac824fcfea65c <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Person> .
_:Bd9c8e7b4a0c5c268362ac824fcfea65c <https://www.ica.org/standards/RiC/ontology#agentIsTargetOfCreationRelation> _:B771bcc5fac4eadc746ae755fdd8d4250 .
_:B0e6a3edcff12b53509f1ce1a0d3150ce <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#CarrierType> .
_:B0e6a3edcff12b53509f1ce1a0d3150ce <https://www.ica.org/standards/RiC/ontology#name> "U-Matic"@de .
_:B3412e1c4c5370642c588bb76aae57107 <https://www.ica.org/standards/RiC/ontology#type> "contributor" .
_:B3412e1c4c5370642c588bb76aae57107 <https://www.ica.org/standards/RiC/ontology#name> "Regie" .
_:B3412e1c4c5370642c588bb76aae57107 <https://www.ica.org/standards/RiC/ontology#creationRelationHasSource> <https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> .
_:B3412e1c4c5370642c588bb76aae57107 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#CreationRelation> .
_:B3412e1c4c5370642c588bb76aae57107 <https://www.ica.org/standards/RiC/ontology#creationRelationHasTarget> _:B72f1d7b60b8f56432c89af2f28ba73a5 .
_:Bea581319X2Dfed1X2D4a4cX2Db77aX2D8b574005427e <https://www.ica.org/standards/RiC/ontology#name> "italienisch" .
_:Bea581319X2Dfed1X2D4a4cX2Db77aX2D8b574005427e <https://www.ica.org/standards/RiC/ontology#type> "content" .
_:Bea581319X2Dfed1X2D4a4cX2Db77aX2D8b574005427e <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Language> .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#scopeAndContent> "Institutionelle Archive und Bestände / Becker Audiovisuals / 2. Tonbildschauen und Multimediaschauen (1968-1997) / 2.2. Bewerbung und Präsentation von Branchen, Institutionen und Produkten / 2.2.2. Firmen und Produkte / 2.2.2.31. Schweizerische Käseunion (1988)"@de .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#isPartOf> <https://memobase.ch/recordSet/BECKER> .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#hasInstantiation> <https://memobase.ch/instantiation/physical/AFZ-IB_Becker_Audiovisuals_69-0> .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#hasSubject> _:B68e6eea7X2D8052X2D4066X2D89a2X2D5d6de7ffb376 .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#hasSubject> _:B810336cfX2Def41X2D4bb1X2Db343X2Dfb8cedfefa11 .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <http://rdaregistry.info/Elements/u/P60441> _:B4e729776cdf4111a444b1b87573ad9af .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#identifiedBy> _:B2be7b778513ddae9a08e689285124ff9 .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#regulatedBy> _:B31be9c84ff352e2fd8760736526ed271 .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#hasInstantiation> <https://memobase.ch/instantiation/digital/AFZ-IB_Becker_Audiovisuals_69-1> .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#type> "Tonbildschau" .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#title> "Die Schweizerische Käseunion AG heisst Sie willkommen in der Swisspack (Film)"@de .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <http://rdaregistry.info/Elements/u/P60451> <https://memobase.ch/institution/Memoriav> .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#hasLanguage> _:Bea581319X2Dfed1X2D4a4cX2Db77aX2D8b574005427e .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#hasSubject> _:B44e300d0X2D255cX2D46e0X2D9416X2Df77d17517eb0 .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Record> .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <http://purl.org/dc/terms/spatial> _:Be19d6f16X2D6b72X2D4cc7X2Dbbc1X2Df229859945f1 .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <http://www.ebu.ch/metadata/ontologies/ebucore/ebucore#hasGenre> _:Becdc1b39X2D102bX2D47dbX2DbdbcX2D406e95f6babd .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#recordResourceOrInstantiationIsSourceOfCreationRelation> _:B5813d2eaad318ca44eb6c808c09d160d .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#hasLanguage> _:Bcb657511X2Db019X2D4b27X2D9f6fX2D0ce00e479bec .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#identifiedBy> _:Bd073dbb0a5b647119528aa7c877142cd .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#recordResourceOrInstantiationIsSourceOfCreationRelation> _:B771bcc5fac4eadc746ae755fdd8d4250 .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <http://purl.org/dc/terms/created> _:Bdf3dfa2af995af00caa8efece83a92cb .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#hasSubject> _:Bbd0dfe5fX2D746dX2D4261X2Db151X2Da98297b648d6 .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#hasTitle> _:B89601cc6c01f7f2f8b327de6357ed403 .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#descriptiveNote> "Wortloser, mit Volksmusik untermalter Industriefilm über die Produktion und Verpackung von Käse durch die Firma Swisspack. Der Film richtet sich an ein internationales Publikum: Im Vorspann werden neben dem deutschen folgende Titel gezeigt: französisch: \"L'Union suisse du commerce de fromage SA vous Souhaite la bienvenue à la Swisspack\", italienisch: \"L'Unione svizzera per il commercio del formaggio SA Vi augura il benvenuto alla Swisspack\", englisch:\"The Swiss Cheese Union Inc. welcomes you to the Swisspack\", spanisch: \"La Unione de los quesos suizos SA augura la vienvenida a la Swisspack\". Überliefert ist: 1 Videokopie (U-Matic)"@de .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#hasSubject> _:Bc83d7a6aX2D609eX2D45ecX2D99dbX2D9a4407b94226 .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#recordResourceOrInstantiationIsSourceOfCreationRelation> _:B3412e1c4c5370642c588bb76aae57107 .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#hasSubject> _:B696014c2X2Dcf3cX2D4d2aX2Dbb13X2D6938339faf3e .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#hasLanguage> _:Bb6135bcdX2D4df0X2D4f28X2D8979X2D5d7f059e19a8 .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#heldBy> <https://memobase.ch/institution/AFZ> .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <http://www.ebu.ch/metadata/ontologies/ebucore/ebucore#hasGenre> _:Bf3aed52fX2D5cb4X2D4c3bX2D8e4eX2D8791378616ed .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <https://www.ica.org/standards/RiC/ontology#hasLanguage> _:Bd232c5b9X2D3a6fX2D4388X2Daa66X2D712b5759cfed .
<https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> <http://schema.org/sameAs> "http://onlinearchives.ethz.ch/load.aspx?guid=a50058d7-9eec-462c-9669-6abb9f26117c" .
_:B696014c2X2Dcf3cX2D4d2aX2Dbb13X2D6938339faf3e <http://www.w3.org/2004/02/skos/core#editorialNote> "Ursprungsfeld: Sachthemen" .
_:B696014c2X2Dcf3cX2D4d2aX2Dbb13X2D6938339faf3e <http://www.w3.org/2004/02/skos/core#prefLabel> "Käseverpackung" .
_:B696014c2X2Dcf3cX2D4d2aX2Dbb13X2D6938339faf3e <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2004/02/skos/core#Concept> .
_:Bd073dbb0a5b647119528aa7c877142cd <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Identifier> .
_:Bd073dbb0a5b647119528aa7c877142cd <https://www.ica.org/standards/RiC/ontology#type> "main" .
_:Bd073dbb0a5b647119528aa7c877142cd <https://www.ica.org/standards/RiC/ontology#identifier> "https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69" .
_:Bc83d7a6aX2D609eX2D45ecX2D99dbX2D9a4407b94226 <https://www.ica.org/standards/RiC/ontology#name> "Swisspack" .
_:Bc83d7a6aX2D609eX2D45ecX2D99dbX2D9a4407b94226 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Agent> .
_:B4e729776cdf4111a444b1b87573ad9af <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#CorporateBody> .
_:B4e729776cdf4111a444b1b87573ad9af <https://www.ica.org/standards/RiC/ontology#name> "Becker Audio-Visuals (BAV)"@de .
_:Ba7bdc4cf2bed30e4524d2dc7f7b56fce <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Identifier> .
_:Ba7bdc4cf2bed30e4524d2dc7f7b56fce <https://www.ica.org/standards/RiC/ontology#type> "main" .
_:Ba7bdc4cf2bed30e4524d2dc7f7b56fce <https://www.ica.org/standards/RiC/ontology#identifier> "https://memobase.ch/instantiation/physical/AFZ-IB_Becker_Audiovisuals_69-0" .
_:B5813d2eaad318ca44eb6c808c09d160d <https://www.ica.org/standards/RiC/ontology#type> "creator" .
_:B5813d2eaad318ca44eb6c808c09d160d <https://www.ica.org/standards/RiC/ontology#name> "Autor" .
_:B5813d2eaad318ca44eb6c808c09d160d <https://www.ica.org/standards/RiC/ontology#creationRelationHasSource> <https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> .
_:B5813d2eaad318ca44eb6c808c09d160d <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#CreationRelation> .
_:B5813d2eaad318ca44eb6c808c09d160d <https://www.ica.org/standards/RiC/ontology#creationRelationHasTarget> _:B3a6874626b84317c29544aac7624a8b2 .
_:Bd232c5b9X2D3a6fX2D4388X2Daa66X2D712b5759cfed <https://www.ica.org/standards/RiC/ontology#name> "englisch" .
_:Bd232c5b9X2D3a6fX2D4388X2Daa66X2D712b5759cfed <https://www.ica.org/standards/RiC/ontology#type> "content" .
_:Bd232c5b9X2D3a6fX2D4388X2Daa66X2D712b5759cfed <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Language> .
_:B44e300d0X2D255cX2D46e0X2D9416X2Df77d17517eb0 <http://www.w3.org/2004/02/skos/core#editorialNote> "Ursprungsfeld: Sachthemen" .
_:B44e300d0X2D255cX2D46e0X2D9416X2Df77d17517eb0 <http://www.w3.org/2004/02/skos/core#prefLabel> "Käseherstellung" .
_:B44e300d0X2D255cX2D46e0X2D9416X2Df77d17517eb0 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2004/02/skos/core#Concept> .
_:B89601cc6c01f7f2f8b327de6357ed403 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Title> .
_:B89601cc6c01f7f2f8b327de6357ed403 <https://www.ica.org/standards/RiC/ontology#type> "main" .
_:B89601cc6c01f7f2f8b327de6357ed403 <https://www.ica.org/standards/RiC/ontology#title> "Die Schweizerische Käseunion AG heisst Sie willkommen in der Swisspack (Film)"@de .
_:Ba8028588863644dfc91b371fa2b2be95 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Identifier> .
_:Ba8028588863644dfc91b371fa2b2be95 <https://www.ica.org/standards/RiC/ontology#type> "main" .
_:Ba8028588863644dfc91b371fa2b2be95 <https://www.ica.org/standards/RiC/ontology#identifier> "https://memobase.ch/instantiation/digital/AFZ-IB_Becker_Audiovisuals_69-1" .
_:B31be9c84ff352e2fd8760736526ed271 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Rule> .
_:B31be9c84ff352e2fd8760736526ed271 <https://www.ica.org/standards/RiC/ontology#type> "holder" .
_:B31be9c84ff352e2fd8760736526ed271 <https://www.ica.org/standards/RiC/ontology#name> "Archiv für Zeitgeschichte (ETH Zürich)" .
_:B31be9c84ff352e2fd8760736526ed271 <https://www.ica.org/standards/RiC/ontology#regulates> <https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> .
_:B68e6eea7X2D8052X2D4066X2D89a2X2D5d6de7ffb376 <https://www.ica.org/standards/RiC/ontology#name> "Käseunion" .
_:B68e6eea7X2D8052X2D4066X2D89a2X2D5d6de7ffb376 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Agent> .
_:B810336cfX2Def41X2D4bb1X2Db343X2Dfb8cedfefa11 <http://www.w3.org/2004/02/skos/core#editorialNote> "Ursprungsfeld: Sachthemen" .
_:B810336cfX2Def41X2D4bb1X2Db343X2Dfb8cedfefa11 <http://www.w3.org/2004/02/skos/core#prefLabel> "Käselagerung" .
_:B810336cfX2Def41X2D4bb1X2Db343X2Dfb8cedfefa11 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2004/02/skos/core#Concept> .
_:Bcb657511X2Db019X2D4b27X2D9f6fX2D0ce00e479bec <https://www.ica.org/standards/RiC/ontology#name> "spanisch" .
_:Bcb657511X2Db019X2D4b27X2D9f6fX2D0ce00e479bec <https://www.ica.org/standards/RiC/ontology#type> "content" .
_:Bcb657511X2Db019X2D4b27X2D9f6fX2D0ce00e479bec <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Language> .
_:Be19d6f16X2D6b72X2D4cc7X2Dbbc1X2Df229859945f1 <https://www.ica.org/standards/RiC/ontology#name> "Schweiz" .
_:Be19d6f16X2D6b72X2D4cc7X2Dbbc1X2Df229859945f1 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Place> .
_:Bb6135bcdX2D4df0X2D4f28X2D8979X2D5d7f059e19a8 <https://www.ica.org/standards/RiC/ontology#name> "deutsch" .
_:Bb6135bcdX2D4df0X2D4f28X2D8979X2D5d7f059e19a8 <https://www.ica.org/standards/RiC/ontology#type> "content" .
_:Bb6135bcdX2D4df0X2D4f28X2D8979X2D5d7f059e19a8 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Language> .
_:B3a6874626b84317c29544aac7624a8b2 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Person> .
_:B3a6874626b84317c29544aac7624a8b2 <https://www.ica.org/standards/RiC/ontology#agentIsTargetOfCreationRelation> _:B5813d2eaad318ca44eb6c808c09d160d .
_:B3a6874626b84317c29544aac7624a8b2 <https://www.ica.org/standards/RiC/ontology#name> "Egon Becker"@de .
_:Becdc1b39X2D102bX2D47dbX2DbdbcX2D406e95f6babd <http://www.w3.org/2004/02/skos/core#editorialNote> "Ursprungsfeld: Genre" .
_:Becdc1b39X2D102bX2D47dbX2DbdbcX2D406e95f6babd <http://www.w3.org/2004/02/skos/core#prefLabel> "Werbung" .
_:Becdc1b39X2D102bX2D47dbX2DbdbcX2D406e95f6babd <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2004/02/skos/core#Concept> .
_:Bf3aed52fX2D5cb4X2D4c3bX2D8e4eX2D8791378616ed <http://www.w3.org/2004/02/skos/core#editorialNote> "Ursprungsfeld: Genre" .
_:Bf3aed52fX2D5cb4X2D4c3bX2D8e4eX2D8791378616ed <http://www.w3.org/2004/02/skos/core#prefLabel> "Image" .
_:Bf3aed52fX2D5cb4X2D4c3bX2D8e4eX2D8791378616ed <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2004/02/skos/core#Concept> .
_:B437ac819636a137448a4457d2016615b <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Rule> .
_:B437ac819636a137448a4457d2016615b <https://www.ica.org/standards/RiC/ontology#type> "usage" .
_:B437ac819636a137448a4457d2016615b <https://www.ica.org/standards/RiC/ontology#name> "In Copyright (InC)" .
_:B437ac819636a137448a4457d2016615b <http://schema.org/sameAs> "http://rightsstatements.org/vocab/InC/1.0/" .
_:B437ac819636a137448a4457d2016615b <https://www.ica.org/standards/RiC/ontology#regulates> <https://memobase.ch/instantiation/digital/AFZ-IB_Becker_Audiovisuals_69-1> .
_:B9e13903910a356a5645cbee68fd40831 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Rule> .
_:B9e13903910a356a5645cbee68fd40831 <https://www.ica.org/standards/RiC/ontology#type> "access" .
_:B9e13903910a356a5645cbee68fd40831 <https://www.ica.org/standards/RiC/ontology#name> "onsite" .
_:B9e13903910a356a5645cbee68fd40831 <https://www.ica.org/standards/RiC/ontology#regulates> <https://memobase.ch/instantiation/physical/AFZ-IB_Becker_Audiovisuals_69-0> .
_:Bbd0dfe5fX2D746dX2D4261X2Db151X2Da98297b648d6 <http://www.w3.org/2004/02/skos/core#editorialNote> "Ursprungsfeld: Sachthemen" .
_:Bbd0dfe5fX2D746dX2D4261X2Db151X2Da98297b648d6 <http://www.w3.org/2004/02/skos/core#prefLabel> "Käseportionierung" .
_:Bbd0dfe5fX2D746dX2D4261X2Db151X2Da98297b648d6 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <http://www.w3.org/2004/02/skos/core#Concept> .
_:B2be7b778513ddae9a08e689285124ff9 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Identifier> .
_:B2be7b778513ddae9a08e689285124ff9 <https://www.ica.org/standards/RiC/ontology#type> "original" .
_:B2be7b778513ddae9a08e689285124ff9 <https://www.ica.org/standards/RiC/ontology#identifier> "IB Becker Audiovisuals / 69" .
_:B1d003c286b7072a7453f0a5cd22701a0 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Identifier> .
_:B1d003c286b7072a7453f0a5cd22701a0 <https://www.ica.org/standards/RiC/ontology#type> "callNumber" .
_:B1d003c286b7072a7453f0a5cd22701a0 <https://www.ica.org/standards/RiC/ontology#identifier> "IB Becker Audiovisuals / 69" .
_:B0d88137eb22e7e5ed6f58c6922f9131d <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Rule> .
_:B0d88137eb22e7e5ed6f58c6922f9131d <https://www.ica.org/standards/RiC/ontology#type> "access" .
_:B0d88137eb22e7e5ed6f58c6922f9131d <https://www.ica.org/standards/RiC/ontology#name> "public" .
_:B0d88137eb22e7e5ed6f58c6922f9131d <https://www.ica.org/standards/RiC/ontology#regulates> <https://memobase.ch/instantiation/digital/AFZ-IB_Becker_Audiovisuals_69-1> .
_:Bdf3dfa2af995af00caa8efece83a92cb <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#DateRange> .
_:Bdf3dfa2af995af00caa8efece83a92cb <https://www.ica.org/standards/RiC/ontology#normalizedDateValue> "1988" .
_:B72f1d7b60b8f56432c89af2f28ba73a5 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Person> .
_:B72f1d7b60b8f56432c89af2f28ba73a5 <https://www.ica.org/standards/RiC/ontology#agentIsTargetOfCreationRelation> _:B3412e1c4c5370642c588bb76aae57107 .
_:B72f1d7b60b8f56432c89af2f28ba73a5 <https://www.ica.org/standards/RiC/ontology#name> "Egon Becker"@de .
_:B771bcc5fac4eadc746ae755fdd8d4250 <https://www.ica.org/standards/RiC/ontology#type> "contributor" .
_:B771bcc5fac4eadc746ae755fdd8d4250 <https://www.ica.org/standards/RiC/ontology#name> "Kamera" .
_:B771bcc5fac4eadc746ae755fdd8d4250 <https://www.ica.org/standards/RiC/ontology#creationRelationHasSource> <https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_69> .
_:B771bcc5fac4eadc746ae755fdd8d4250 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#CreationRelation> .
_:B771bcc5fac4eadc746ae755fdd8d4250 <https://www.ica.org/standards/RiC/ontology#creationRelationHasTarget> _:Bd9c8e7b4a0c5c268362ac824fcfea65c .
This is a testfile.
\ No newline at end of file
......@@ -21,6 +21,7 @@ import java.io.Closeable
import java.io.File
import java.io.StringWriter
import java.net.URI
import org.apache.jena.rdf.model.Model
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.apache.kafka.clients.consumer.ConsumerRecords
import org.apache.logging.log4j.LogManager
......@@ -32,7 +33,8 @@ import org.memobase.sftp.SftpClient
class Ingester(
private val producer: Producer,
private val sftpClient: SftpClient,
private val fedoraClient: FedoraClient
private val fedoraClient: FedoraClient,
private val externalBaseUrl: String
) : Closeable {
private val log = LogManager.getLogger("FedoraIngester")
......@@ -48,23 +50,58 @@ class Ingester(
private fun ingest(id: String, content: String) {
log.info("Ingest record $id.")
log.info("Content: $content")
val rdfHandler = RdfHandler(content)
val rdfHandler = RdfHandler(content, externalBaseUrl)
val recordOutput = StringWriter()
val recordPair = rdfHandler.getRecord()
recordPair.second.write(recordOutput, "NTRIPLES")
// FIXME: correct for ingest:
// 1 binaries
// 2 instantiations
// 3 records
val data = recordOutput.toString()
log.info("Ingesting record ${recordPair.first}.")
// create placeholders referenced resources:
val nonBinaryResources = rdfHandler.getReferencedNonBinaryResources()
nonBinaryResources.forEach { resource ->
log.info("Creating placeholder for resource $resource.")
fedoraClient.createPlaceholder(URI(resource))
log.info("Created placeholder for resource $resource.")
}
// ingest record, instantiations and binaries:
fedoraClient.createOrUpdateRdfResource(URI(recordPair.first), data, RdfContentTypes.NTRIPLES)
ingestInstantiations(rdfHandler.getInstantiations())
ingestBinaries(rdfHandler.getSftpLocators(), rdfHandler)
log.info("Ingested record ${recordPair.first}.")
}
// FIXME: extract into separate method
rdfHandler.getInstantiations().forEach { instantiationPair ->
private fun processRecord(record: ConsumerRecord<String, String>): Report {
if (record.value().contains("ERROR")) {
return Report(
id = record.key(),
status = ReportStatus.failure,
message = ReportMessages.ingestFailed(record.key())
)
}
return try {
ingest(record.key(), record.value())
Report(
id = record.key(),
status = ReportStatus.success,
message = ReportMessages.ingestedRecord(record.key())
)
} catch (e: FcrepoOperationFailedException) {
log.error("Ingestion of record ${record.key()} failed: ${e.localizedMessage}.")
Report(
id = record.key(),
status = ReportStatus.failure,
message = ReportMessages.ingestFailed(record.key())
)
}
}
private fun ingestInstantiations(instantiations: List<Pair<String, Model>>) {
instantiations.forEach { instantiationPair ->
val instantiationOutput = StringWriter()
instantiationPair.second.write(instantiationOutput, "NTRIPLES")
try {
......@@ -77,9 +114,10 @@ class Ingester(
throw ex
}
}
}
// FIXME: extract into separate method
rdfHandler.getSftpLocators().forEach {
private fun ingestBinaries(sftpLocators: List<Pair<String, String?>>, rdfHandler: RdfHandler) {
sftpLocators.forEach {
val digitalInstantiationUrl = it.first
it.second.let { path ->
if (path != null) {
......@@ -101,36 +139,8 @@ class Ingester(
}
}
private fun processRecord(record: ConsumerRecord<String, String>): Report {
if (record.value().contains("ERROR")) {
return Report(
id = record.key(),
status = ReportStatus.failure,
message = ReportMessages.ingestFailed(record.key())
)
}
return try {
ingest(record.key(), record.value())
Report(
id = record.key(),
status = ReportStatus.success,
message = ReportMessages.ingestedRecord(record.key())
)
} catch (e: FcrepoOperationFailedException) {
log.error("Ingestion of record ${record.key()} failed: ${e.localizedMessage}.")
Report(
id = record.key(),
status = ReportStatus.failure,
message = ReportMessages.ingestFailed(record.key())
)
}
}
override fun close() {
sftpClient.close()
producer.close()
// TODO: the fedora client should probably be closeable as well?
// fedora.close()
}
}
......@@ -11,9 +11,10 @@ import org.memobase.rdf.EBUCORE
import org.memobase.rdf.RDF
import org.memobase.rdf.RICO
class RdfHandler(data: String) {
class RdfHandler(data: String, externalBaseUrl: String) {
private val log = LogManager.getLogger("RdfHandler")
private val model = ModelFactory.createDefaultModel()
private val externalBaseUrl = externalBaseUrl
init {
model.read(ByteArrayInputStream(data.toByteArray()), "", "NTRIPLES")
......@@ -43,6 +44,20 @@ class RdfHandler(data: String) {
return Pair(uri, resultModel)
}
private fun isInternalNonBinaryResource(uri: String): Boolean {
return uri.startsWith(externalBaseUrl) && !uri.endsWith("/binary")
}
fun getReferencedNonBinaryResources(): List<String> {
return model.listStatements().filterKeep { statement ->
statement.getObject().isURIResource
}.mapWith { statement ->
statement.getObject().asResource().uri
}.filterKeep { uri ->
isInternalNonBinaryResource(uri)
}.toList().distinct()
}
fun getInstantiations(): List<Pair<String, Model>> {
return model.listSubjectsWithProperty(RDF.type, RICO.Instantiation).mapWith { resource ->
val model = ModelFactory.createDefaultModel()
......
......@@ -58,7 +58,8 @@ class Service(fileName: String = "app.yml") {
private val producer = Producer(settings.kafkaProducerSettings, settings.outputTopic)
private val sftp = SftpClient(settings.sftpSettings)
private val fedora = createFedoraClient(settings.appSettings)
val ingester = Ingester(producer, sftp, fedora)
val ingester = Ingester(producer, sftp, fedora,
settings.appSettings.getProperty("$FEDORA_PROPERTIES_PREFIX.externalBaseUrl"))
fun run() {
consumer.use { consumer ->
......
......@@ -31,7 +31,7 @@ class TestRdfHandler {
@Test
fun `test get record`() {
val rdfHandler = RdfHandler(readFile("input.nt"))
val rdfHandler = RdfHandler(readFile("input.nt"), "https://memobase.ch/")
val pair = rdfHandler.getRecord()
......@@ -47,9 +47,29 @@ class TestRdfHandler {
.isEqualTo("https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_63")
}
@Test
fun `test get referenced non binary resources`() {
val rdfHandler = RdfHandler(readFile("input.nt"), "https://memobase.ch/")
val resources = rdfHandler.getReferencedNonBinaryResources()
resources.forEachIndexed { index, uri ->
assertThat(uri)
.isEqualTo(nonBinaryResourcesResult[index])
}
}
private val nonBinaryResourcesResult: List<String> = listOf(
"https://memobase.ch/instantiation/physical/AFZ-IB_Becker_Audiovisuals_63-0",
"https://memobase.ch/institution/AFZ",
"https://memobase.ch/recordSet/BECKER",
"https://memobase.ch/institution/Memoriav",
"https://memobase.ch/instantiation/digital/AFZ-IB_Becker_Audiovisuals_63-1",
"https://memobase.ch/record/AFZ-IB_Becker_Audiovisuals_63"
)
@Test
fun `test get instantiations`() {
val rdfHandler = RdfHandler(readFile("input.nt"))
val rdfHandler = RdfHandler(readFile("input.nt"), "https://memobase.ch/")
val list = rdfHandler.getInstantiations()
list.forEachIndexed { index, pair ->
......@@ -72,7 +92,7 @@ class TestRdfHandler {
@Test
fun `test get sftp locators`() {
val rdfHandler = RdfHandler(readFile("inputSftp.nt"))
val rdfHandler = RdfHandler(readFile("inputSftp.nt"), "https://memobase.ch/")
val pairs = rdfHandler.getSftpLocators()
pairs.forEachIndexed { index, pair ->
......@@ -88,7 +108,7 @@ class TestRdfHandler {
@Test
fun `test replace sftp locators`() {
val rdfHandler = RdfHandler(readFile("inputSftp.nt"))
val rdfHandler = RdfHandler(readFile("inputSftp.nt"), "https://memobase.ch/")
val list = rdfHandler.getInstantiations()
list.forEachIndexed { index, pair ->
......@@ -106,7 +126,7 @@ class TestRdfHandler {
@Test
fun `test mime type extraction for binaries`() {
val rdfHandler = RdfHandler(readFile("input.nt"))
val rdfHandler = RdfHandler(readFile("input.nt"), "https://memobase.ch/")
assertThat(rdfHandler.getMimeType().first().second).isEqualTo("video/mpeg")
}
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment