Due to a scheduled upgrade to version 14.10, GitLab will be unavailabe on Monday 30.05., from 19:00 until 20:00.

Commit b964e3ee authored by Thomas Bernhart's avatar Thomas Bernhart
Browse files

Upgrade to fedora-client 0.7.0, harmonize error reporting

parent f2a19ea0
Pipeline #22020 passed with stages
in 3 minutes and 59 seconds
......@@ -47,7 +47,7 @@ dependencies {
implementation 'org.apache.jena:apache-jena:3.14.0', excludeSlf4J
implementation 'org.memobase:memobase-service-utilities:2.0.2', excludeSlf4J
implementation 'org.memobase:fedora-client:0.6.2', excludeSlf4J
implementation 'ch.memobase:fedora-client:0.7.0', excludeSlf4J
// KOTLIN IMPORTS
implementation 'org.jetbrains.kotlin:kotlin-stdlib-jdk8'
......
......@@ -17,18 +17,22 @@
*/
package org.memobase
import ch.memobase.exceptions.SftpClientException
import ch.memobase.fedora.FedoraClient
import ch.memobase.fedora.FedoraTransactionClient
import ch.memobase.fedora.RdfContentTypes
import ch.memobase.sftp.SftpClient
import java.io.File
import java.io.IOException
import java.io.StringWriter
import java.net.URI
import java.net.URISyntaxException
import org.apache.jena.rdf.model.Model
import org.apache.jena.riot.Lang
import org.apache.jena.riot.RDFDataMgr
import org.apache.logging.log4j.LogManager
import org.fcrepo.client.FcrepoOperationFailedException
import org.memobase.fedora.FedoraClient
import org.memobase.fedora.FedoraTransactionClient
import org.memobase.fedora.RdfContentTypes
import org.memobase.exceptions.MissingMimeTypeException
class Ingester(
private val sftpClient: SftpClient,
......@@ -38,7 +42,7 @@ class Ingester(
private val log = LogManager.getLogger("FedoraIngester")
@Throws(FcrepoOperationFailedException::class)
@Throws(FcrepoOperationFailedException::class, IOException::class, MissingMimeTypeException::class, SftpClientException::class, URISyntaxException::class)
fun ingest(id: String, content: String) {
log.info("Begin ingest of message with id $id.")
val rdfHandler = RdfHandler(content, externalBaseUrl)
......@@ -70,6 +74,7 @@ class Ingester(
log.info("End ingest of message with id $id.")
}
@Throws(FcrepoOperationFailedException::class, IOException::class, URISyntaxException::class)
private fun ingestInstantiations(instantiations: List<Pair<String, Model>>, transaction: FedoraTransactionClient) {
instantiations.forEach { instantiationPair ->
val instantiationOutput = StringWriter()
......@@ -85,6 +90,7 @@ class Ingester(
}
}
@Throws(FcrepoOperationFailedException::class, IOException::class, MissingMimeTypeException::class, SftpClientException::class, URISyntaxException::class)
private fun ingestBinaries(
sftpLocators: List<Pair<String, String?>>,
rdfHandler: RdfHandler,
......
......@@ -123,6 +123,7 @@ class RdfHandler(data: String, private val externalBaseUrl: String) {
}.toList()
}
@Throws(MissingMimeTypeException::class)
fun getMimeType(uri: String): String {
val mimeTypes = model.getResource(uri).listProperties(EBUCORE.hasMimeType).mapWith {
it.`object`.asLiteral().string
......
......@@ -19,18 +19,20 @@
package org.memobase
import ch.memobase.exceptions.SftpClientException
import ch.memobase.fedora.FedoraClient
import ch.memobase.fedora.FedoraClientImpl
import ch.memobase.reporting.ReportStatus
import ch.memobase.settings.SettingsLoader
import ch.memobase.sftp.SftpClient
import java.io.Closeable
import java.io.IOException
import java.net.URISyntaxException
import java.util.Properties
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.apache.logging.log4j.LogManager
import org.apache.logging.log4j.Logger
import org.fcrepo.client.FcrepoOperationFailedException
import org.memobase.exceptions.MissingMimeTypeException
import org.memobase.fedora.FedoraClient
import org.memobase.fedora.FedoraClientImpl
class Service(fileName: String = "app.yml") : Closeable {
......@@ -121,54 +123,99 @@ class Service(fileName: String = "app.yml") : Closeable {
} catch (ex: FcrepoOperationFailedException) {
log.error(ex.localizedMessage)
Report(
key, ReportStatus.fatal, ex.localizedMessage
id = record.key(),
status = ReportStatus.fatal,
message = "Unable to write to Fedora: ${ex.localizedMessage}"
)
} catch (ex: IOException) {
log.error("${ex.javaClass.canonicalName}: ${ex.localizedMessage}", ex)
Report(
id = record.key(),
status = ReportStatus.fatal,
message = "IO error: ${ex.localizedMessage}"
)
} catch (ex: URISyntaxException) {
log.error("${ex.javaClass.canonicalName}: ${ex.localizedMessage}", ex)
Report(
id = record.key(),
status = ReportStatus.fatal,
message = "Malformed URI: ${ex.localizedMessage}"
)
} catch (ex: Exception) {
log.error("${ex.javaClass.canonicalName}: ${ex.localizedMessage}", ex)
Report(
id = record.key(),
status = ReportStatus.fatal,
message = "Unknown error: ${ex.localizedMessage}"
)
}
}
private fun processRecord(record: ConsumerRecord<String, String>): Report {
val ingester = sftpClient?.let {
Ingester(
it,
fedoraClient,
settings.appSettings.getProperty("$FEDORA_PROPERTIES_PREFIX.externalBaseUrl")
)
}
val key = record.key()
val value = record.value()
if (key == null)
return Report("NoKey", ReportStatus.fatal, "The key in message is null.")
if (value == null)
return Report(key, ReportStatus.fatal, "The value in message is null.")
return try {
ingester?.ingest(record.key(), record.value())
val ingester = sftpClient?.let {
Ingester(
it,
fedoraClient,
settings.appSettings.getProperty("$FEDORA_PROPERTIES_PREFIX.externalBaseUrl")
)
}
ingester?.ingest(key, value)
Report(
id = record.key(),
id = key,
status = ReportStatus.success,
message = ReportMessages.ingestedRecord(record.key())
message = ReportMessages.ingestedRecord(key)
)
} catch (ex: FcrepoOperationFailedException) {
log.error("${ex.javaClass.canonicalName}:: ${ex.localizedMessage}", ex)
log.error("${ex.javaClass.canonicalName}: ${ex.localizedMessage}", ex)
Report(
id = record.key(),
id = key,
status = ReportStatus.fatal,
message = "Fedora Exception: ${ex.localizedMessage}"
message = "Unable to write to Fedora: ${ex.localizedMessage}"
)
} catch (ex: IOException) {
log.error("${ex.javaClass.canonicalName}: ${ex.localizedMessage}", ex)
Report(
id = key,
status = ReportStatus.fatal,
message = "IO error: ${ex.localizedMessage}"
)
} catch (ex: MissingMimeTypeException) {
log.error("${ex.javaClass.canonicalName}:: ${ex.localizedMessage}", ex)
log.error("${ex.javaClass.canonicalName}: ${ex.localizedMessage}", ex)
Report(
id = record.key(),
id = key,
status = ReportStatus.fatal,
message = "Missing MimeType Exception: ${ex.localizedMessage}"
message = "Missing MimeType: ${ex.localizedMessage}"
)
} catch (ex: SftpClientException) {
log.error("${ex.javaClass.canonicalName}: ${ex.localizedMessage}", ex)
Report(
id = record.key(),
id = key,
status = ReportStatus.fatal,
message = "SFTP Exception: ${ex.localizedMessage}"
message = "Unable to read from SFTP: ${ex.localizedMessage}"
)
} catch (ex: URISyntaxException) {
log.error("${ex.javaClass.canonicalName}: ${ex.localizedMessage}", ex)
Report(
id = key,
status = ReportStatus.fatal,
message = "Malformed URI: ${ex.localizedMessage}"
)
} catch (ex: Exception) {
log.error("${ex.javaClass.canonicalName}: ${ex.localizedMessage}", ex)
Report(
id = record.key(),
id = key,
status = ReportStatus.fatal,
message = "Unknown Exception: ${ex.localizedMessage}"
message = "Unknown error: ${ex.localizedMessage}"
)
}
}
......
......@@ -17,18 +17,20 @@
*/
package org.memobase
import ch.memobase.fedora.FedoraClient
import ch.memobase.fedora.RdfContentTypes
import java.io.IOException
import java.net.URI
import java.net.URISyntaxException
import org.apache.logging.log4j.LogManager
import org.fcrepo.client.FcrepoOperationFailedException
import org.memobase.fedora.FedoraClient
import org.memobase.fedora.RdfContentTypes
class SimpleIngester(
private val fedoraClient: FedoraClient
) {
private val log = LogManager.getLogger("SimpleIngester")
@Throws(FcrepoOperationFailedException::class)
@Throws(FcrepoOperationFailedException::class, IOException::class, URISyntaxException::class)
fun ingest(id: String, content: String) {
fedoraClient.startTransaction().use { transaction ->
log.info("Begin transaction to ingest single entity with uri $id.")
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment