Commit 86b718b3 authored by Matthias's avatar Matthias
Browse files

some steps for adding thumbnails (code breaks!)

parent 379962dc
Pipeline #15571 failed with stages
in 1 minute and 33 seconds
...@@ -47,8 +47,7 @@ dependencies { ...@@ -47,8 +47,7 @@ dependencies {
// JSON Parser // JSON Parser
implementation 'com.beust:klaxon:5.2' implementation 'com.beust:klaxon:5.2'
implementation 'org.memobase:memobase-service-utilities:0.14.2' implementation 'org.memobase:memobase-service-utilities:0.16.0'
implementation 'ch.memobase:import-process-effects-registry_2.12:0.2.1'
implementation 'org.apache.jena:apache-jena:3.14.0' implementation 'org.apache.jena:apache-jena:3.14.0'
......
...@@ -21,6 +21,7 @@ package org.memobase ...@@ -21,6 +21,7 @@ package org.memobase
import org.apache.jena.rdf.model.Model import org.apache.jena.rdf.model.Model
import org.apache.jena.rdf.model.ModelFactory import org.apache.jena.rdf.model.ModelFactory
import org.apache.jena.rdf.model.Resource import org.apache.jena.rdf.model.Resource
import org.apache.jena.rdf.model.ResourceFactory
import org.apache.jena.rdf.model.impl.StatementImpl import org.apache.jena.rdf.model.impl.StatementImpl
import org.apache.kafka.streams.StreamsBuilder import org.apache.kafka.streams.StreamsBuilder
import org.apache.kafka.streams.kstream.Predicate import org.apache.kafka.streams.kstream.Predicate
...@@ -32,6 +33,8 @@ import org.memobase.reports.ReportMessages ...@@ -32,6 +33,8 @@ import org.memobase.reports.ReportMessages
import org.memobase.reports.ReportStatus import org.memobase.reports.ReportStatus
import org.memobase.settings.SettingsLoader import org.memobase.settings.SettingsLoader
import org.memobase.sftp.SftpClient import org.memobase.sftp.SftpClient
import settings.HeaderExtractionTransformSupplier
import settings.HeaderMetadata
import java.io.StringReader import java.io.StringReader
import java.io.StringWriter import java.io.StringWriter
import kotlin.system.exitProcess import kotlin.system.exitProcess
...@@ -50,21 +53,20 @@ class KafkaTopology(private val settings: SettingsLoader) { ...@@ -50,21 +53,20 @@ class KafkaTopology(private val settings: SettingsLoader) {
private val sftpClient = SftpClient(settings.sftpSettings) private val sftpClient = SftpClient(settings.sftpSettings)
private val sftpBasePath = appSettings.getProperty(Constant.sftpBasePathPropertyName) private val sftpBasePath = appSettings.getProperty(Constant.sftpBasePathPropertyName)
private val files = createMediaFileList(settings.appSettings.getProperty(Constant.recordSetIdPropertyName))
private fun createMediaFileList(setting: String): List<String> { private fun createMediaFileList(setting: String): List<String> {
return try { return try {
val list = sftpClient.listFiles("$sftpBasePath/$setting/${Constant.mediaFolderName}") val list = sftpClient.listFiles("$sftpBasePath/$setting/${Constant.mediaFolderName}")
log.info("Files found on sftp server: $list.") log.info("Files found on sftp server: $list.")
list list
} catch (ex: Exception) { } catch (ex: Exception) {
//TODO: Report erstellen
ex.printStackTrace() ex.printStackTrace()
log.error(ex.localizedMessage + ": $sftpBasePath/$setting/${Constant.mediaFolderName}") log.error(ex.localizedMessage + ": $sftpBasePath/$setting/${Constant.mediaFolderName}")
exitProcess(1) exitProcess(1)
} }
} }
private val reportingTopic = "${settings.processReportTopic}-${Constant.topicReportingSuffix}" private val reportingTopic = "${settings.processReportTopic}"
fun prepare(): StreamsBuilder { fun prepare(): StreamsBuilder {
val builder = StreamsBuilder() val builder = StreamsBuilder()
...@@ -72,8 +74,10 @@ class KafkaTopology(private val settings: SettingsLoader) { ...@@ -72,8 +74,10 @@ class KafkaTopology(private val settings: SettingsLoader) {
val stream = builder.stream<String, String>(settings.inputTopic) val stream = builder.stream<String, String>(settings.inputTopic)
val instantiationBranch = stream val instantiationBranch = stream
.transformValues(HeaderExtractionTransformSupplier<String>())
.mapValues { value -> createModel(value) } .mapValues { value -> createModel(value) }
.mapValues { value -> extractSubjects(value) } .mapValues { value -> extractSubjects(value) }
.mapValues { value -> createThumbnail(value) }
.branch( .branch(
Predicate { _, value -> containsDigitalObjectWithoutLocator(value.second) }, Predicate { _, value -> containsDigitalObjectWithoutLocator(value.second) },
Predicate { _, _ -> true } Predicate { _, _ -> true }
...@@ -97,7 +101,6 @@ class KafkaTopology(private val settings: SettingsLoader) { ...@@ -97,7 +101,6 @@ class KafkaTopology(private val settings: SettingsLoader) {
ReportStatus.success, ReportStatus.success,
"Create sftp file path to instantiation." "Create sftp file path to instantiation."
).toJson() ).toJson()
} }
.to(reportingTopic) .to(reportingTopic)
...@@ -118,8 +121,8 @@ class KafkaTopology(private val settings: SettingsLoader) { ...@@ -118,8 +121,8 @@ class KafkaTopology(private val settings: SettingsLoader) {
return builder return builder
} }
private fun extractSubjects(model: Model): Pair<Model, List<Resource>> { private fun extractSubjects(input: Pair<Model, HeaderMetadata>): Pair<Pair<Model, HeaderMetadata>, List<Resource>> {
return Pair(model, model.listSubjects().toList()) return Pair(input, input.first.listSubjects().toList())
} }
private fun containsDigitalObjectWithoutLocator(res: List<Resource>): Boolean { private fun containsDigitalObjectWithoutLocator(res: List<Resource>): Boolean {
...@@ -135,13 +138,13 @@ class KafkaTopology(private val settings: SettingsLoader) { ...@@ -135,13 +138,13 @@ class KafkaTopology(private val settings: SettingsLoader) {
} }
} }
private fun createModel(data: String): Model { private fun createModel(data: Pair<String, HeaderMetadata>): Pair<Model, HeaderMetadata> {
val model = ModelFactory.createDefaultModel() val model = ModelFactory.createDefaultModel()
model.read(StringReader(data), "", Constant.rdfParserLang) model.read(StringReader(data.first), "", Constant.rdfParserLang)
return model return Pair(model, data.second)
} }
private fun enrichSftpLocator(key: String, data: Pair<Model, List<Resource>>): Pair<Model, Report> { private fun enrichSftpLocator(key: String, data: Pair<Pair<Model, HeaderMetadata>, List<Resource>>): Pair<Model, Report> {
var link = "" var link = ""
val digitalObject = val digitalObject =
data.second.first { it.hasProperty(RICO.type, Constant.digitalObject) && !it.hasProperty(EBUCORE.locator) } data.second.first { it.hasProperty(RICO.type, Constant.digitalObject) && !it.hasProperty(EBUCORE.locator) }
...@@ -149,16 +152,21 @@ class KafkaTopology(private val settings: SettingsLoader) { ...@@ -149,16 +152,21 @@ class KafkaTopology(private val settings: SettingsLoader) {
val originalIdentifier = try { val originalIdentifier = try {
getOriginalIdentifiers(data.second)[0] getOriginalIdentifiers(data.second)[0]
} catch (ex: IndexOutOfBoundsException) { } catch (ex: IndexOutOfBoundsException) {
return Pair(data.first, Report(key, ReportStatus.failure, ReportMessages.noOriginalIdentifier(key))) return Pair(data.first.first, Report(key, ReportStatus.failure, ReportMessages.noOriginalIdentifier(key)))
} }
val value = originalIdentifier.getProperty(RICO.identifier) val value = originalIdentifier.getProperty(RICO.identifier).string
val files = createMediaFileList(data.first.second.recordSetId)
// TODO: instead of looping 'files', loop only given uri below with a fixed set of extensions
sftpClient.exists(data.first.second.recordSetId + "/" + value + ".jpg")
for (file in files) { for (file in files) {
if (file.contains(value.string)) { if (file.contains(value)) {
link = "${Constant.sftpPathPrefix}$file" link = "${Constant.sftpPathPrefix}$file"
val literal = data.first.createLiteral(link) val literal = ResourceFactory.createPlainLiteral(link)
data.first.add(StatementImpl(digitalObject, EBUCORE.locator, literal)) digitalObject.addLiteral(EBUCORE.locator, literal)
return Pair( return Pair(
data.first, data.first.first,
Report( Report(
digitalObject.uri, digitalObject.uri,
ReportStatus.success, ReportStatus.success,
...@@ -168,7 +176,7 @@ class KafkaTopology(private val settings: SettingsLoader) { ...@@ -168,7 +176,7 @@ class KafkaTopology(private val settings: SettingsLoader) {
} }
} }
return Pair( return Pair(
data.first, data.first.first,
Report( Report(
digitalObject.uri, digitalObject.uri,
ReportStatus.failure, ReportStatus.failure,
...@@ -176,4 +184,15 @@ class KafkaTopology(private val settings: SettingsLoader) { ...@@ -176,4 +184,15 @@ class KafkaTopology(private val settings: SettingsLoader) {
) )
) )
} }
private fun createThumbnail(key: String, data: Pair<Pair<Model, HeaderMetadata>, List<Resource>>): Pair<Model, Report> {
// TODO: same here as in enrichSftpLocator but for thum
val thumbnail = data.first.first.createResource()
thumbnail.addProperty(RDF.type, RICO.Instantiation)
thumbnail.addProperty(RICO.type, "thumbnail")
thumbnail.addProperty(RICO.type, "thumbnail")
}
} }
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment