Commit 0dadac04 authored by Jonas Waeber's avatar Jonas Waeber
Browse files

Update service util dependency

Replace report with report implementation from dependency
parent 0eb999ec
......@@ -46,7 +46,7 @@ dependencies {
// JSON Parser
implementation 'com.beust:klaxon:5.2'
implementation 'org.memobase:memobase-service-utilities:2.0.4'
implementation 'org.memobase:memobase-service-utilities:2.0.15'
implementation 'org.apache.jena:apache-jena:3.14.0'
......
......@@ -13,4 +13,6 @@ object Constant {
const val thumbnailRicoType = "thumbnail"
const val sftpPathPrefix = "sftp:"
const val serviceName = "media-linker"
}
......@@ -18,40 +18,11 @@
package org.memobase
import ch.memobase.reporting.Report
import com.beust.klaxon.Klaxon
import java.time.LocalDateTime
data class FinalReport(
val id: String,
val status: String,
val message: String,
val step: String = "media-linker",
val timestamp: String = LocalDateTime.now().toString()
) {
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (javaClass != other?.javaClass) return false
other as FinalReport
if (id != other.id) return false
if (status != other.status) return false
if (message != other.message) return false
if (step != other.step) return false
return true
}
override fun hashCode(): Int {
var result = id.hashCode()
result = 31 * result + status.hashCode()
result = 31 * result + message.hashCode()
result = 31 * result + step.hashCode()
return result
}
}
data class Report(
data class IntermediateReport(
val id: String,
val status: String,
val generalFailureMessage: String = "",
......@@ -66,7 +37,7 @@ data class Report(
(if (digitalObjectMessage != "") "DIGITAL OBJECT: $digitalObjectMessage" else "") +
(if (thumbnailMessage != "") " -- THUMBNAIL: $thumbnailMessage" else "")
}
val finalReport = FinalReport(this.id, this.status, message)
val finalReport = Report(this.id, this.status, message, Constant.serviceName)
return Klaxon().toJsonString(finalReport)
}
}
......@@ -21,14 +21,11 @@ package org.memobase
import ch.memobase.rdf.EBUCORE
import ch.memobase.rdf.RDF
import ch.memobase.rdf.RICO
import ch.memobase.reporting.ReportStatus
import ch.memobase.settings.HeaderExtractionTransformSupplier
import ch.memobase.settings.HeaderMetadata
import ch.memobase.settings.SettingsLoader
import ch.memobase.sftp.SftpClient
import java.io.StringReader
import java.io.StringWriter
import java.net.MalformedURLException
import java.net.URL
import org.apache.jena.rdf.model.Model
import org.apache.jena.rdf.model.ModelFactory
import org.apache.jena.rdf.model.Resource
......@@ -39,7 +36,10 @@ import org.apache.kafka.streams.kstream.KStream
import org.apache.kafka.streams.kstream.Predicate
import org.apache.logging.log4j.LogManager
import org.memobase.reports.ReportMessages
import org.memobase.reports.ReportStatus
import java.io.StringReader
import java.io.StringWriter
import java.net.MalformedURLException
import java.net.URL
class KafkaTopology(private val settings: SettingsLoader) {
private val appSettings = settings.appSettings
......@@ -67,7 +67,7 @@ class KafkaTopology(private val settings: SettingsLoader) {
model[1]
.mapValues { key, _ ->
log.warn("Invalid input data. Check mapper service processing.")
Report(
IntermediateReport(
key,
ReportStatus.fatal,
generalFailureMessage = "Invalid input data. Check mapper service processing."
......@@ -82,7 +82,7 @@ class KafkaTopology(private val settings: SettingsLoader) {
createRecord(
value,
key,
ReportStatus.ignore,
ReportStatus.ignored,
digitalObjectMessage = "No digital object resource present in model."
)
} else {
......@@ -113,7 +113,7 @@ class KafkaTopology(private val settings: SettingsLoader) {
}
.branch(
Predicate { _, value -> value.third.status == ReportStatus.fatal },
Predicate { _, value -> value.third.status == ReportStatus.ignore },
Predicate { _, value -> value.third.status == ReportStatus.ignored },
Predicate { _, _ -> true }
)
......@@ -210,7 +210,7 @@ class KafkaTopology(private val settings: SettingsLoader) {
return builder
}
private fun fetchThumbnailForYoutubeOrVimeoFile(value: Triple<Pair<Model, HeaderMetadata>, List<Resource>, Report>): Triple<Pair<Model, HeaderMetadata>, List<Resource>, Report> {
private fun fetchThumbnailForYoutubeOrVimeoFile(value: Triple<Pair<Model, HeaderMetadata>, List<Resource>, IntermediateReport>): Triple<Pair<Model, HeaderMetadata>, List<Resource>, IntermediateReport> {
if (noThumbnailAttached(value.second)) {
val recordResource = getRecordResource(value.second)
val digitalObjectResource = getDigitalObjectResource(value.second)
......@@ -251,7 +251,7 @@ class KafkaTopology(private val settings: SettingsLoader) {
log.debug("Won't fetch thumbnail file for ${value.third.id} because no youtube/vimeo resource")
return updateRecord(
value,
ReportStatus.ignore,
ReportStatus.ignored,
thumbnailMessage = "no thumbnails available"
)
}
......@@ -275,11 +275,11 @@ class KafkaTopology(private val settings: SettingsLoader) {
}
private fun addLocalThumbnail(
value: Triple<Pair<Model, HeaderMetadata>, List<Resource>, Report>,
value: Triple<Pair<Model, HeaderMetadata>, List<Resource>, IntermediateReport>,
recordResource: Resource,
digitalObjectResource: Resource,
pathToLocalFile: String
): Triple<Pair<Model, HeaderMetadata>, List<Resource>, Report> {
): Triple<Pair<Model, HeaderMetadata>, List<Resource>, IntermediateReport> {
val destPath = "$sftpBasePath/${value.first.second.recordSetId}/${Constant.thumbnailFolderName}/${
recordResource.uri.split(
"/"
......@@ -311,9 +311,9 @@ class KafkaTopology(private val settings: SettingsLoader) {
}
private fun addDimensionsToDigitalObject(
value: Triple<Pair<Model, HeaderMetadata>, List<Resource>, Report>,
value: Triple<Pair<Model, HeaderMetadata>, List<Resource>, IntermediateReport>,
oembedObject: OembedResponse
): Triple<Pair<Model, HeaderMetadata>, List<Resource>, Report> {
): Triple<Pair<Model, HeaderMetadata>, List<Resource>, IntermediateReport> {
val digitalObjectResource = getDigitalObjectResource(value.second)!!
if (oembedObject.width != null) {
val width = ResourceFactory.createPlainLiteral(oembedObject.width.toString())
......@@ -338,7 +338,7 @@ class KafkaTopology(private val settings: SettingsLoader) {
private fun hasAccessibleDigitalObjectWithoutLocator(res: List<Resource>): Boolean {
return res.any {
it.hasProperty(RICO.type, Constant.digitalObject) && !it.hasProperty(EBUCORE.locator) &&
it.hasProperty(RICO.regulatedBy) && !isFaroResource(res)
it.hasProperty(RICO.regulatedBy) && !isFaroResource(res)
}
}
......@@ -360,8 +360,8 @@ class KafkaTopology(private val settings: SettingsLoader) {
private fun hasNoAccessProperty(res: List<Resource>): Boolean {
return res.any {
it.hasProperty(RICO.type, Constant.digitalObject) &&
!it.hasProperty(EBUCORE.locator) &&
!it.hasProperty(RICO.regulatedBy)
!it.hasProperty(EBUCORE.locator) &&
!it.hasProperty(RICO.regulatedBy)
}
}
......@@ -406,29 +406,29 @@ class KafkaTopology(private val settings: SettingsLoader) {
private fun addThumbnailSftpLocatorToModel(
key: String,
data: Triple<Pair<Model, HeaderMetadata>, List<Resource>, Report>
): Triple<Pair<Model, HeaderMetadata>, List<Resource>, Report> {
data: Triple<Pair<Model, HeaderMetadata>, List<Resource>, IntermediateReport>
): Triple<Pair<Model, HeaderMetadata>, List<Resource>, IntermediateReport> {
return addSftpLocatorToModel(key, data, Constant.thumbnailFolderName)
}
private fun addMediaSftpLocatorToModel(
key: String,
data: Triple<Pair<Model, HeaderMetadata>, List<Resource>, Report>
): Triple<Pair<Model, HeaderMetadata>, List<Resource>, Report> {
data: Triple<Pair<Model, HeaderMetadata>, List<Resource>, IntermediateReport>
): Triple<Pair<Model, HeaderMetadata>, List<Resource>, IntermediateReport> {
return addSftpLocatorToModel(key, data, Constant.mediaFolderName)
}
private fun addSftpLocatorToModel(
key: String,
data: Triple<Pair<Model, HeaderMetadata>, List<Resource>, Report>,
data: Triple<Pair<Model, HeaderMetadata>, List<Resource>, IntermediateReport>,
type: String
): Triple<Pair<Model, HeaderMetadata>, List<Resource>, Report> {
): Triple<Pair<Model, HeaderMetadata>, List<Resource>, IntermediateReport> {
val recordResource = getRecordResource(data.second)!!
val digitalObjectResource = getDigitalObjectResource(data.second)!!
val originalIdentifierValue = getOriginalIdentifier(recordResource)!!
val link = getLinkToResourceOnSFTPServer(data.first.second.recordSetId, type, originalIdentifierValue)
?: return if (type == Constant.thumbnailFolderName) {
updateRecord(data, ReportStatus.ignore, thumbnailMessage = "no local thumbnails available")
updateRecord(data, ReportStatus.ignored, thumbnailMessage = "no local thumbnails available")
} else {
updateRecord(data, ReportStatus.warning, digitalObjectMessage = ReportMessages.reportFailure(key, type))
}
......@@ -494,12 +494,12 @@ class KafkaTopology(private val settings: SettingsLoader) {
}
private fun updateRecord(
value: Triple<Pair<Model, HeaderMetadata>, List<Resource>, Report>,
value: Triple<Pair<Model, HeaderMetadata>, List<Resource>, IntermediateReport>,
status: String,
generalMessage: String = "",
digitalObjectMessage: String = "",
thumbnailMessage: String = ""
): Triple<Pair<Model, HeaderMetadata>, List<Resource>, Report> {
): Triple<Pair<Model, HeaderMetadata>, List<Resource>, IntermediateReport> {
val amendedReport = value.third.copy(
status = status,
generalFailureMessage = if (generalMessage != "") generalMessage else value.third.generalFailureMessage,
......@@ -516,8 +516,8 @@ class KafkaTopology(private val settings: SettingsLoader) {
generalFailureMessage: String = "",
digitalObjectMessage: String = "",
thumbnailMessage: String = ""
): Triple<Pair<Model, HeaderMetadata>, List<Resource>, Report> {
val report = Report(
): Triple<Pair<Model, HeaderMetadata>, List<Resource>, IntermediateReport> {
val report = IntermediateReport(
messageId,
status,
generalFailureMessage = generalFailureMessage,
......@@ -527,13 +527,13 @@ class KafkaTopology(private val settings: SettingsLoader) {
return updateRecord(Triple(value.first, value.second, report), status = status)
}
private fun report(kstream: KStream<String, Triple<Pair<Model, HeaderMetadata>, List<Resource>, Report>>) {
private fun report(kstream: KStream<String, Triple<Pair<Model, HeaderMetadata>, List<Resource>, IntermediateReport>>) {
kstream
.mapValues { _, value -> value.third.toJson() }
.to(reportingTopic)
}
private fun sendDownstream(kstream: KStream<String, Triple<Pair<Model, HeaderMetadata>, List<Resource>, Report>>) {
private fun sendDownstream(kstream: KStream<String, Triple<Pair<Model, HeaderMetadata>, List<Resource>, IntermediateReport>>) {
kstream
.mapValues { value -> serializeModel(value.first.first) }
.to(settings.outputTopic)
......
package org.memobase.reports
object ReportStatus {
const val success = "SUCCESS"
const val ignore = "IGNORE"
const val warning = "WARNING"
const val fatal = "FATAL"
}
......@@ -141,7 +141,7 @@ class TestKafkaTopology {
)
val data = reportedRecord.value()
val report = Klaxon().parse<Report>(data)
val report = Klaxon().parse<IntermediateReport>(data)
assertAll(
"",
......@@ -156,7 +156,7 @@ class TestKafkaTopology {
},
{
assertThat(report)
.isEqualTo(Klaxon().parse<Report>(readFile("output${params.count}.json")))
.isEqualTo(Klaxon().parse<IntermediateReport>(readFile("output${params.count}.json")))
}
)
}
......
{
"id": "https://memobase.ch/record/test-institution-id-MEI_49887",
"status": "IGNORE",
"status": "IGNORED",
"message": "faro resources don't have accessible media files"
}
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment