Commit 76237d1a authored by Matthias's avatar Matthias
Browse files

make thumbnails work

parent 9da0d4d2
Pipeline #15854 passed with stages
in 5 minutes and 39 seconds
......@@ -52,20 +52,6 @@ class KafkaTopology(private val settings: SettingsLoader) {
)
private val sftpClient = SftpClient(settings.sftpSettings)
private val sftpBasePath = appSettings.getProperty(Constant.sftpBasePathPropertyName)
private fun createMediaFileList(setting: String): List<String> {
return try {
val list = sftpClient.listFiles("$sftpBasePath/$setting/${Constant.mediaFolderName}")
log.info("Files found on sftp server: $list.")
list
} catch (ex: Exception) {
//TODO: Report erstellen
ex.printStackTrace()
log.error(ex.localizedMessage + ": $sftpBasePath/$setting/${Constant.mediaFolderName}")
exitProcess(1)
}
}
private val reportingTopic = "${settings.processReportTopic}"
fun prepare(): StreamsBuilder {
......@@ -77,7 +63,13 @@ class KafkaTopology(private val settings: SettingsLoader) {
.transformValues(HeaderExtractionTransformSupplier<String>())
.mapValues { value -> createModel(value) }
.mapValues { value -> extractSubjects(value) }
//.mapValues { value -> createThumbnail(value) }
.mapValues { value ->
enrichSftpLocator(
"key",
Triple(value.first, value.second, Report("", "", "")),
"thumbnails"
)
}
.branch(
Predicate { _, value -> containsDigitalObjectWithoutLocator(value.second) },
Predicate { _, _ -> true }
......@@ -87,7 +79,7 @@ class KafkaTopology(private val settings: SettingsLoader) {
.mapValues { readOnlyKey, value -> enrichSftpLocator(readOnlyKey, value) }
.mapValues { value ->
val out = StringWriter()
value.first.write(out, Constant.rdfParserLang)
value.first.first.write(out, Constant.rdfParserLang)
out.toString().trim()
}
......@@ -144,7 +136,11 @@ class KafkaTopology(private val settings: SettingsLoader) {
return Pair(model, data.second)
}
private fun enrichSftpLocator(key: String, data: Pair<Pair<Model, HeaderMetadata>, List<Resource>>): Pair<Model, Report> {
private fun enrichSftpLocator(
key: String,
data: Triple<Pair<Model, HeaderMetadata>, List<Resource>, Report>,
type: String = "media"
): Triple<Pair<Model, HeaderMetadata>, List<Resource>, Report> {
var link = ""
val digitalObject =
data.second.first { it.hasProperty(RICO.type, Constant.digitalObject) && !it.hasProperty(EBUCORE.locator) }
......@@ -152,50 +148,59 @@ class KafkaTopology(private val settings: SettingsLoader) {
val originalIdentifier = try {
getOriginalIdentifiers(data.second)[0]
} catch (ex: IndexOutOfBoundsException) {
return Pair(data.first.first, Report(key, ReportStatus.failure, ReportMessages.noOriginalIdentifier(key)))
return Triple(
data.first,
data.second,
Report(key, ReportStatus.failure, ReportMessages.noOriginalIdentifier(key))
)
}
val value = originalIdentifier.getProperty(RICO.identifier).string
val record =
data.second.first { it.hasProperty(RDF.type, RICO.Record) }
val fileExtensions = arrayOf("jpg", "jpeg", "png", "mp3", "mp4")
for (extension in fileExtensions) {
val filePath = sftpBasePath + "/" + data.first.second.recordSetId + "/media/" + value + "." + extension
val filePath = "$sftpBasePath/${data.first.second.recordSetId}/$type/$value.$extension"
if (sftpClient.exists(filePath)) {
link = "${Constant.sftpPathPrefix}$filePath"
val literal = ResourceFactory.createPlainLiteral(link)
digitalObject.addLiteral(EBUCORE.locator, literal)
data.first.first.createLiteral(digitalObject.toString(), true)
return Pair(
data.first.first,
if (type == "media") {
val literal = ResourceFactory.createPlainLiteral(link)
digitalObject.addLiteral(EBUCORE.locator, literal)
data.first.first.createLiteral(digitalObject.toString(), true)
} else if (type == "thumbnails") {
val thumbnail = data.first.first.createResource(
"https://memobase.ch/digital/${digitalObject.uri.substringAfterLast("/")}/derived"
)
val literal = ResourceFactory.createPlainLiteral(link)
thumbnail.addProperty(RDF.type, RICO.Instantiation)
thumbnail.addProperty(RICO.type, "thumbnail")
thumbnail.addProperty(EBUCORE.locator, literal)
digitalObject.addProperty(RICO.hasDerivedInstantiation, thumbnail)
thumbnail.addProperty(RICO.isDerivedFromInstantiation, digitalObject)
record.addProperty(RICO.hasInstantiation, thumbnail)
thumbnail.addProperty(RICO.instantiates, record)
}
return Triple(
data.first,
data.second,
Report(
digitalObject.uri,
record.uri,
ReportStatus.success,
ReportMessages.reportSuccess(digitalObject.uri, link)
data.third.message + "\n" + ReportMessages.reportSuccess(digitalObject.uri, link)
)
)
break
}
}
return Pair(
data.first.first,
return Triple(
data.first,
data.second,
Report(
digitalObject.uri,
record.uri,
ReportStatus.failure,
ReportMessages.reportFailure(digitalObject.uri)
data.third.message + "\n" + ReportMessages.reportSuccess(digitalObject.uri, link)
)
)
}
/*
private fun createThumbnail(key: data: Pair<Pair<Model, HeaderMetadata>, List<Resource>>): Pair<Model, Report> {
// TODO: same here as in enrichSftpLocator but for thum
val thumbnail = data.first.first.createResource()
thumbnail.addProperty(RDF.type, RICO.Instantiation)
thumbnail.addProperty(RICO.type, "thumbnail")
thumbnail.addProperty(EBUCORE.locator, "locator of file")
return Pair(data.first.first, new Report("id00", "undefined","just a sample report"))
}
*/
}
......@@ -22,8 +22,8 @@ import com.beust.klaxon.Klaxon
data class Report(
val id: String,
val status: String,
val message: String
val status: String, // success even if thumbnail is failue
val message: String // "ok, thumbnail created" oder "ok, no thumbnail"
) {
fun toJson(): String {
return Klaxon().toJsonString(this)
......
......@@ -54,18 +54,24 @@ class TestKafkaTopology {
private val sftpServer = EmbeddedSftpServer(22000, "user", "password")
init {
val files = listOf(
val media = listOf(
Pair("/base/test_record_set_1/media", "MEI_49884.jpg"),
Pair("/base/test_record_set_1/media", "MEI_49885.jpg"),
Pair("/base/test_record_set_1/media", "MEI_49886.jpg"),
Pair("/base/test_record_set_1/thumbnails", "MEI_49884_thumbnail.jpg"),
Pair("/base/test_record_set_1/thumbnails", "MEI_49885_thumbnail.jpg"),
Pair("/base/test_record_set_1/thumbnails", "MEI_49886_thumbnail.jpg")
Pair("/base/test_record_set_1/media", "MEI_49886.jpg")
)
for (pair in files) {
val thumbnails = listOf(
Pair("/base/test_record_set_1/thumbnails", "MEI_49884.jpg"),
Pair("/base/test_record_set_1/thumbnails", "MEI_49885.jpg"),
Pair("/base/test_record_set_1/thumbnails", "MEI_49886.jpg")
)
for (pair in media) {
sftpServer.putFile(Paths.get(pair.first, pair.second).toString(), FileInputStream(Paths.get("src/test/resources/data", pair.second).toFile()))
}
for (pair in thumbnails) {
sftpServer.putFile(Paths.get(pair.first, pair.second).toString(), FileInputStream(Paths.get("src/test/resources/data/thumbnails", pair.second).toFile()))
}
}
@ParameterizedTest
......@@ -137,13 +143,13 @@ class TestKafkaTopology {
"output1.nt",
"output1.json"
),
TestParams(
"test1.yml",
"https://memobase.ch/instantiation/physical/BAZ-MEI_49885-0",
"input2.nt",
"https://memobase.ch/instantiation/physical/BAZ-MEI_49885-0",
"output2.nt",
"output2.json"
)
TestParams(
"test1.yml",
"https://memobase.ch/instantiation/physical/BAZ-MEI_49885-0",
"input2.nt",
"https://memobase.ch/instantiation/physical/BAZ-MEI_49885-0",
"output2.nt",
"output2.json"
)
)
}
......@@ -20,4 +20,11 @@ _:N995c0ab9252144edb4fca436615e30b4 <http://www.w3.org/1999/02/22-rdf-syntax-ns#
<https://memobase.ch/instantiation/digital/BAZ-MEI_49885-1> <https://www.ica.org/standards/RiC/ontology#isDerivedFromInstantiation> <https://memobase.ch/instantiation/physical/BAZ-MEI_49885-0> .
_:N995c0ab9252144edb4fca436615e30b4 <https://www.ica.org/standards/RiC/ontology#type> "main" .
_:Nd1834b21ce544129a81ea909a410c533 <https://www.ica.org/standards/RiC/ontology#type> "original" .
_:N8e2f5cb72415481aad5a88cdd85a6325 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Rule> .
\ No newline at end of file
_:N8e2f5cb72415481aad5a88cdd85a6325 <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Rule> .
<https://memobase.ch/record/test-institution-id-MEI_49884> <http://memobase.ch/internal/isPublished> "false"^^<http://www.w3.org/2001/XMLSchema#boolean> .
<https://memobase.ch/record/test-institution-id-MEI_49884> <http://rdaregistry.info/Elements/u/P60451> <https://memobase.ch/institution/mrv> .
<https://memobase.ch/record/test-institution-id-MEI_49884> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Record> .
<https://memobase.ch/record/test-institution-id-MEI_49884> <https://www.ica.org/standards/RiC/ontology#hasInstantiation> <https://memobase.ch/digital/BAZ-MEI_49885-1> .
<https://memobase.ch/record/test-institution-id-MEI_49884> <https://www.ica.org/standards/RiC/ontology#heldBy> <https://memobase.ch/institution/test-record-set-id> .
<https://memobase.ch/record/test-institution-id-MEI_49884> <https://www.ica.org/standards/RiC/ontology#isPartOf> <https://memobase.ch/recordSet/test-institution-id> .
<https://memobase.ch/record/test-institution-id-MEI_49884> <https://www.ica.org/standards/RiC/ontology#type> "Foto" .
<https://memobase.ch/instantiation/physical/BAZ-MEI_49885-0> <https://www.ica.org/standards/RiC/ontology#type> "physicalObject" .
<https://memobase.ch/instantiation/physical/BAZ-MEI_49885-0> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Instantiation> .
\ No newline at end of file
<https://memobase.ch/instantiation/physical/BAZ-MEI_49885-0> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Instantiation> .
<https://memobase.ch/instantiation/digital/BAZ-MEI_49885-0> <https://www.ica.org/standards/RiC/ontology#type> "digitalObject" .
<https://memobase.ch/instantiation/digital/BAZ-MEI_49885-0> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Instantiation> .
<https://memobase.ch/record/test-institution-id-MEI_49884> <http://memobase.ch/internal/isPublished> "false"^^<http://www.w3.org/2001/XMLSchema#boolean> .
<https://memobase.ch/record/test-institution-id-MEI_49884> <http://rdaregistry.info/Elements/u/P60451> <https://memobase.ch/institution/mrv> .
<https://memobase.ch/record/test-institution-id-MEI_49884> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Record> .
<https://memobase.ch/record/test-institution-id-MEI_49884> <https://www.ica.org/standards/RiC/ontology#hasInstantiation> <https://memobase.ch/digital/BAZ-MEI_49885-1> .
<https://memobase.ch/record/test-institution-id-MEI_49884> <https://www.ica.org/standards/RiC/ontology#heldBy> <https://memobase.ch/institution/test-record-set-id> .
<https://memobase.ch/record/test-institution-id-MEI_49884> <https://www.ica.org/standards/RiC/ontology#isPartOf> <https://memobase.ch/recordSet/test-institution-id> .
<https://memobase.ch/record/test-institution-id-MEI_49884> <https://www.ica.org/standards/RiC/ontology#type> "Foto" .
<https://memobase.ch/digital/BAZ-MEI_49885-1/derived> <http://www.ebu.ch/metadata/ontologies/ebucore/ebucore#locator> "sftp:/base/test_record_set_1/thumbnails/MEI_49885.jpg" .
<https://memobase.ch/digital/BAZ-MEI_49885-1/derived> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Instantiation> .
<https://memobase.ch/digital/BAZ-MEI_49885-1/derived> <https://www.ica.org/standards/RiC/ontology#instantiates> <https://memobase.ch/record/test-institution-id-MEI_49884> .
<https://memobase.ch/digital/BAZ-MEI_49885-1/derived> <https://www.ica.org/standards/RiC/ontology#isDerivedFromInstantiation> <https://memobase.ch/instantiation/digital/BAZ-MEI_49885-1> .
<https://memobase.ch/digital/BAZ-MEI_49885-1/derived> <https://www.ica.org/standards/RiC/ontology#type> "thumbnail" .
<https://memobase.ch/instantiation/digital/BAZ-MEI_49885-1> <http://www.ebu.ch/metadata/ontologies/ebucore/ebucore#locator> "sftp:/base/test_record_set_1/media/MEI_49885.jpg" .
<https://memobase.ch/instantiation/digital/BAZ-MEI_49885-1> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Instantiation> .
<https://memobase.ch/instantiation/digital/BAZ-MEI_49885-1> <https://www.ica.org/standards/RiC/ontology#hasDerivedInstantiation> <https://memobase.ch/digital/BAZ-MEI_49885-1/derived> .
<https://memobase.ch/instantiation/digital/BAZ-MEI_49885-1> <https://www.ica.org/standards/RiC/ontology#identifiedBy> _:B .
<https://memobase.ch/instantiation/digital/BAZ-MEI_49885-1> <https://www.ica.org/standards/RiC/ontology#identifiedBy> _:B .
<https://memobase.ch/instantiation/digital/BAZ-MEI_49885-1> <https://www.ica.org/standards/RiC/ontology#instantiates> <https://memobase.ch/record/BAZ-MEI_49885> .
......@@ -7,6 +13,14 @@
<https://memobase.ch/instantiation/digital/BAZ-MEI_49885-1> <https://www.ica.org/standards/RiC/ontology#regulatedBy> _:B .
<https://memobase.ch/instantiation/digital/BAZ-MEI_49885-1> <https://www.ica.org/standards/RiC/ontology#regulatedBy> _:B .
<https://memobase.ch/instantiation/digital/BAZ-MEI_49885-1> <https://www.ica.org/standards/RiC/ontology#type> "digitalObject" .
<https://memobase.ch/record/test-institution-id-MEI_49884> <http://memobase.ch/internal/isPublished> "false"^^<http://www.w3.org/2001/XMLSchema#boolean> .
<https://memobase.ch/record/test-institution-id-MEI_49884> <http://rdaregistry.info/Elements/u/P60451> <https://memobase.ch/institution/mrv> .
<https://memobase.ch/record/test-institution-id-MEI_49884> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Record> .
<https://memobase.ch/record/test-institution-id-MEI_49884> <https://www.ica.org/standards/RiC/ontology#hasInstantiation> <https://memobase.ch/digital/BAZ-MEI_49885-1/derived> .
<https://memobase.ch/record/test-institution-id-MEI_49884> <https://www.ica.org/standards/RiC/ontology#hasInstantiation> <https://memobase.ch/digital/BAZ-MEI_49885-1> .
<https://memobase.ch/record/test-institution-id-MEI_49884> <https://www.ica.org/standards/RiC/ontology#heldBy> <https://memobase.ch/institution/test-record-set-id> .
<https://memobase.ch/record/test-institution-id-MEI_49884> <https://www.ica.org/standards/RiC/ontology#isPartOf> <https://memobase.ch/recordSet/test-institution-id> .
<https://memobase.ch/record/test-institution-id-MEI_49884> <https://www.ica.org/standards/RiC/ontology#type> "Foto" .
_:B <http://schema.org/sameAs> "http://rightsstatements.org/vocab/InC/1.0/" .
_:B <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Identifier> .
_:B <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Identifier> .
......
<https://memobase.ch/instantiation/digital/BAZ-MEI_49885-0> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Instantiation> .
<https://memobase.ch/instantiation/digital/BAZ-MEI_49885-0> <https://www.ica.org/standards/RiC/ontology#type> "digitalObject" .
<https://memobase.ch/instantiation/physical/BAZ-MEI_49885-0> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Instantiation> .
<https://memobase.ch/instantiation/physical/BAZ-MEI_49885-0> <https://www.ica.org/standards/RiC/ontology#type> "physicalObject" .
\ No newline at end of file
<https://memobase.ch/instantiation/physical/BAZ-MEI_49885-0> <https://www.ica.org/standards/RiC/ontology#type> "physicalObject" .
<https://memobase.ch/record/test-institution-id-MEI_49884> <http://memobase.ch/internal/isPublished> "false"^^<http://www.w3.org/2001/XMLSchema#boolean> .
<https://memobase.ch/record/test-institution-id-MEI_49884> <http://rdaregistry.info/Elements/u/P60451> <https://memobase.ch/institution/mrv> .
<https://memobase.ch/record/test-institution-id-MEI_49884> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Record> .
<https://memobase.ch/record/test-institution-id-MEI_49884> <https://www.ica.org/standards/RiC/ontology#hasInstantiation> <https://memobase.ch/digital/BAZ-MEI_49885-1> .
<https://memobase.ch/record/test-institution-id-MEI_49884> <https://www.ica.org/standards/RiC/ontology#heldBy> <https://memobase.ch/institution/test-record-set-id> .
<https://memobase.ch/record/test-institution-id-MEI_49884> <https://www.ica.org/standards/RiC/ontology#isPartOf> <https://memobase.ch/recordSet/test-institution-id> .
<https://memobase.ch/record/test-institution-id-MEI_49884> <https://www.ica.org/standards/RiC/ontology#type> "Foto" .
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment