Commit b964e3ee authored by Thomas Bernhart's avatar Thomas Bernhart
Browse files

Upgrade to fedora-client 0.7.0, harmonize error reporting

parent f2a19ea0
Pipeline #22020 passed with stages
in 3 minutes and 59 seconds
...@@ -47,7 +47,7 @@ dependencies { ...@@ -47,7 +47,7 @@ dependencies {
implementation 'org.apache.jena:apache-jena:3.14.0', excludeSlf4J implementation 'org.apache.jena:apache-jena:3.14.0', excludeSlf4J
implementation 'org.memobase:memobase-service-utilities:2.0.2', excludeSlf4J implementation 'org.memobase:memobase-service-utilities:2.0.2', excludeSlf4J
implementation 'org.memobase:fedora-client:0.6.2', excludeSlf4J implementation 'ch.memobase:fedora-client:0.7.0', excludeSlf4J
// KOTLIN IMPORTS // KOTLIN IMPORTS
implementation 'org.jetbrains.kotlin:kotlin-stdlib-jdk8' implementation 'org.jetbrains.kotlin:kotlin-stdlib-jdk8'
......
...@@ -17,18 +17,22 @@ ...@@ -17,18 +17,22 @@
*/ */
package org.memobase package org.memobase
import ch.memobase.exceptions.SftpClientException
import ch.memobase.fedora.FedoraClient
import ch.memobase.fedora.FedoraTransactionClient
import ch.memobase.fedora.RdfContentTypes
import ch.memobase.sftp.SftpClient import ch.memobase.sftp.SftpClient
import java.io.File import java.io.File
import java.io.IOException
import java.io.StringWriter import java.io.StringWriter
import java.net.URI import java.net.URI
import java.net.URISyntaxException
import org.apache.jena.rdf.model.Model import org.apache.jena.rdf.model.Model
import org.apache.jena.riot.Lang import org.apache.jena.riot.Lang
import org.apache.jena.riot.RDFDataMgr import org.apache.jena.riot.RDFDataMgr
import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.LogManager
import org.fcrepo.client.FcrepoOperationFailedException import org.fcrepo.client.FcrepoOperationFailedException
import org.memobase.fedora.FedoraClient import org.memobase.exceptions.MissingMimeTypeException
import org.memobase.fedora.FedoraTransactionClient
import org.memobase.fedora.RdfContentTypes
class Ingester( class Ingester(
private val sftpClient: SftpClient, private val sftpClient: SftpClient,
...@@ -38,7 +42,7 @@ class Ingester( ...@@ -38,7 +42,7 @@ class Ingester(
private val log = LogManager.getLogger("FedoraIngester") private val log = LogManager.getLogger("FedoraIngester")
@Throws(FcrepoOperationFailedException::class) @Throws(FcrepoOperationFailedException::class, IOException::class, MissingMimeTypeException::class, SftpClientException::class, URISyntaxException::class)
fun ingest(id: String, content: String) { fun ingest(id: String, content: String) {
log.info("Begin ingest of message with id $id.") log.info("Begin ingest of message with id $id.")
val rdfHandler = RdfHandler(content, externalBaseUrl) val rdfHandler = RdfHandler(content, externalBaseUrl)
...@@ -70,6 +74,7 @@ class Ingester( ...@@ -70,6 +74,7 @@ class Ingester(
log.info("End ingest of message with id $id.") log.info("End ingest of message with id $id.")
} }
@Throws(FcrepoOperationFailedException::class, IOException::class, URISyntaxException::class)
private fun ingestInstantiations(instantiations: List<Pair<String, Model>>, transaction: FedoraTransactionClient) { private fun ingestInstantiations(instantiations: List<Pair<String, Model>>, transaction: FedoraTransactionClient) {
instantiations.forEach { instantiationPair -> instantiations.forEach { instantiationPair ->
val instantiationOutput = StringWriter() val instantiationOutput = StringWriter()
...@@ -85,6 +90,7 @@ class Ingester( ...@@ -85,6 +90,7 @@ class Ingester(
} }
} }
@Throws(FcrepoOperationFailedException::class, IOException::class, MissingMimeTypeException::class, SftpClientException::class, URISyntaxException::class)
private fun ingestBinaries( private fun ingestBinaries(
sftpLocators: List<Pair<String, String?>>, sftpLocators: List<Pair<String, String?>>,
rdfHandler: RdfHandler, rdfHandler: RdfHandler,
......
...@@ -123,6 +123,7 @@ class RdfHandler(data: String, private val externalBaseUrl: String) { ...@@ -123,6 +123,7 @@ class RdfHandler(data: String, private val externalBaseUrl: String) {
}.toList() }.toList()
} }
@Throws(MissingMimeTypeException::class)
fun getMimeType(uri: String): String { fun getMimeType(uri: String): String {
val mimeTypes = model.getResource(uri).listProperties(EBUCORE.hasMimeType).mapWith { val mimeTypes = model.getResource(uri).listProperties(EBUCORE.hasMimeType).mapWith {
it.`object`.asLiteral().string it.`object`.asLiteral().string
......
...@@ -19,18 +19,20 @@ ...@@ -19,18 +19,20 @@
package org.memobase package org.memobase
import ch.memobase.exceptions.SftpClientException import ch.memobase.exceptions.SftpClientException
import ch.memobase.fedora.FedoraClient
import ch.memobase.fedora.FedoraClientImpl
import ch.memobase.reporting.ReportStatus import ch.memobase.reporting.ReportStatus
import ch.memobase.settings.SettingsLoader import ch.memobase.settings.SettingsLoader
import ch.memobase.sftp.SftpClient import ch.memobase.sftp.SftpClient
import java.io.Closeable import java.io.Closeable
import java.io.IOException
import java.net.URISyntaxException
import java.util.Properties import java.util.Properties
import org.apache.kafka.clients.consumer.ConsumerRecord import org.apache.kafka.clients.consumer.ConsumerRecord
import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.LogManager
import org.apache.logging.log4j.Logger import org.apache.logging.log4j.Logger
import org.fcrepo.client.FcrepoOperationFailedException import org.fcrepo.client.FcrepoOperationFailedException
import org.memobase.exceptions.MissingMimeTypeException import org.memobase.exceptions.MissingMimeTypeException
import org.memobase.fedora.FedoraClient
import org.memobase.fedora.FedoraClientImpl
class Service(fileName: String = "app.yml") : Closeable { class Service(fileName: String = "app.yml") : Closeable {
...@@ -121,12 +123,45 @@ class Service(fileName: String = "app.yml") : Closeable { ...@@ -121,12 +123,45 @@ class Service(fileName: String = "app.yml") : Closeable {
} catch (ex: FcrepoOperationFailedException) { } catch (ex: FcrepoOperationFailedException) {
log.error(ex.localizedMessage) log.error(ex.localizedMessage)
Report( Report(
key, ReportStatus.fatal, ex.localizedMessage id = record.key(),
status = ReportStatus.fatal,
message = "Unable to write to Fedora: ${ex.localizedMessage}"
)
} catch (ex: IOException) {
log.error("${ex.javaClass.canonicalName}: ${ex.localizedMessage}", ex)
Report(
id = record.key(),
status = ReportStatus.fatal,
message = "IO error: ${ex.localizedMessage}"
)
} catch (ex: URISyntaxException) {
log.error("${ex.javaClass.canonicalName}: ${ex.localizedMessage}", ex)
Report(
id = record.key(),
status = ReportStatus.fatal,
message = "Malformed URI: ${ex.localizedMessage}"
)
} catch (ex: Exception) {
log.error("${ex.javaClass.canonicalName}: ${ex.localizedMessage}", ex)
Report(
id = record.key(),
status = ReportStatus.fatal,
message = "Unknown error: ${ex.localizedMessage}"
) )
} }
} }
private fun processRecord(record: ConsumerRecord<String, String>): Report { private fun processRecord(record: ConsumerRecord<String, String>): Report {
val key = record.key()
val value = record.value()
if (key == null)
return Report("NoKey", ReportStatus.fatal, "The key in message is null.")
if (value == null)
return Report(key, ReportStatus.fatal, "The value in message is null.")
return try {
val ingester = sftpClient?.let { val ingester = sftpClient?.let {
Ingester( Ingester(
it, it,
...@@ -134,41 +169,53 @@ class Service(fileName: String = "app.yml") : Closeable { ...@@ -134,41 +169,53 @@ class Service(fileName: String = "app.yml") : Closeable {
settings.appSettings.getProperty("$FEDORA_PROPERTIES_PREFIX.externalBaseUrl") settings.appSettings.getProperty("$FEDORA_PROPERTIES_PREFIX.externalBaseUrl")
) )
} }
ingester?.ingest(key, value)
return try {
ingester?.ingest(record.key(), record.value())
Report( Report(
id = record.key(), id = key,
status = ReportStatus.success, status = ReportStatus.success,
message = ReportMessages.ingestedRecord(record.key()) message = ReportMessages.ingestedRecord(key)
) )
} catch (ex: FcrepoOperationFailedException) { } catch (ex: FcrepoOperationFailedException) {
log.error("${ex.javaClass.canonicalName}:: ${ex.localizedMessage}", ex) log.error("${ex.javaClass.canonicalName}: ${ex.localizedMessage}", ex)
Report( Report(
id = record.key(), id = key,
status = ReportStatus.fatal,
message = "Unable to write to Fedora: ${ex.localizedMessage}"
)
} catch (ex: IOException) {
log.error("${ex.javaClass.canonicalName}: ${ex.localizedMessage}", ex)
Report(
id = key,
status = ReportStatus.fatal, status = ReportStatus.fatal,
message = "Fedora Exception: ${ex.localizedMessage}" message = "IO error: ${ex.localizedMessage}"
) )
} catch (ex: MissingMimeTypeException) { } catch (ex: MissingMimeTypeException) {
log.error("${ex.javaClass.canonicalName}:: ${ex.localizedMessage}", ex) log.error("${ex.javaClass.canonicalName}: ${ex.localizedMessage}", ex)
Report( Report(
id = record.key(), id = key,
status = ReportStatus.fatal, status = ReportStatus.fatal,
message = "Missing MimeType Exception: ${ex.localizedMessage}" message = "Missing MimeType: ${ex.localizedMessage}"
) )
} catch (ex: SftpClientException) { } catch (ex: SftpClientException) {
log.error("${ex.javaClass.canonicalName}: ${ex.localizedMessage}", ex) log.error("${ex.javaClass.canonicalName}: ${ex.localizedMessage}", ex)
Report( Report(
id = record.key(), id = key,
status = ReportStatus.fatal, status = ReportStatus.fatal,
message = "SFTP Exception: ${ex.localizedMessage}" message = "Unable to read from SFTP: ${ex.localizedMessage}"
)
} catch (ex: URISyntaxException) {
log.error("${ex.javaClass.canonicalName}: ${ex.localizedMessage}", ex)
Report(
id = key,
status = ReportStatus.fatal,
message = "Malformed URI: ${ex.localizedMessage}"
) )
} catch (ex: Exception) { } catch (ex: Exception) {
log.error("${ex.javaClass.canonicalName}: ${ex.localizedMessage}", ex) log.error("${ex.javaClass.canonicalName}: ${ex.localizedMessage}", ex)
Report( Report(
id = record.key(), id = key,
status = ReportStatus.fatal, status = ReportStatus.fatal,
message = "Unknown Exception: ${ex.localizedMessage}" message = "Unknown error: ${ex.localizedMessage}"
) )
} }
} }
......
...@@ -17,18 +17,20 @@ ...@@ -17,18 +17,20 @@
*/ */
package org.memobase package org.memobase
import ch.memobase.fedora.FedoraClient
import ch.memobase.fedora.RdfContentTypes
import java.io.IOException
import java.net.URI import java.net.URI
import java.net.URISyntaxException
import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.LogManager
import org.fcrepo.client.FcrepoOperationFailedException import org.fcrepo.client.FcrepoOperationFailedException
import org.memobase.fedora.FedoraClient
import org.memobase.fedora.RdfContentTypes
class SimpleIngester( class SimpleIngester(
private val fedoraClient: FedoraClient private val fedoraClient: FedoraClient
) { ) {
private val log = LogManager.getLogger("SimpleIngester") private val log = LogManager.getLogger("SimpleIngester")
@Throws(FcrepoOperationFailedException::class) @Throws(FcrepoOperationFailedException::class, IOException::class, URISyntaxException::class)
fun ingest(id: String, content: String) { fun ingest(id: String, content: String) {
fedoraClient.startTransaction().use { transaction -> fedoraClient.startTransaction().use { transaction ->
log.info("Begin transaction to ingest single entity with uri $id.") log.info("Begin transaction to ingest single entity with uri $id.")
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment