Commit 9f925305 authored by Jonas Waeber's avatar Jonas Waeber
Browse files

Update utility dependency to 2.0.2.

Failures are now fatal reports.
parent 61db65ac
...@@ -45,7 +45,7 @@ dependencies { ...@@ -45,7 +45,7 @@ dependencies {
implementation 'org.apache.jena:apache-jena:3.14.0', excludeSlf4J implementation 'org.apache.jena:apache-jena:3.14.0', excludeSlf4J
implementation 'org.memobase:memobase-service-utilities:1.12.2', excludeSlf4J implementation 'org.memobase:memobase-service-utilities:2.0.2', excludeSlf4J
implementation 'org.memobase:fedora-client:0.6.2', excludeSlf4J implementation 'org.memobase:fedora-client:0.6.2', excludeSlf4J
// KOTLIN IMPORTS // KOTLIN IMPORTS
......
...@@ -17,9 +17,7 @@ ...@@ -17,9 +17,7 @@
*/ */
package org.memobase package org.memobase
import java.io.File import ch.memobase.sftp.SftpClient
import java.io.StringWriter
import java.net.URI
import org.apache.jena.rdf.model.Model import org.apache.jena.rdf.model.Model
import org.apache.jena.riot.Lang import org.apache.jena.riot.Lang
import org.apache.jena.riot.RDFDataMgr import org.apache.jena.riot.RDFDataMgr
...@@ -28,7 +26,9 @@ import org.fcrepo.client.FcrepoOperationFailedException ...@@ -28,7 +26,9 @@ import org.fcrepo.client.FcrepoOperationFailedException
import org.memobase.fedora.FedoraClient import org.memobase.fedora.FedoraClient
import org.memobase.fedora.FedoraTransactionClient import org.memobase.fedora.FedoraTransactionClient
import org.memobase.fedora.RdfContentTypes import org.memobase.fedora.RdfContentTypes
import org.memobase.sftp.SftpClient import java.io.File
import java.io.StringWriter
import java.net.URI
class Ingester( class Ingester(
private val sftpClient: SftpClient, private val sftpClient: SftpClient,
...@@ -76,12 +76,20 @@ class Ingester( ...@@ -76,12 +76,20 @@ class Ingester(
RDFDataMgr.write(instantiationOutput, instantiationPair.second, Lang.NTRIPLES) RDFDataMgr.write(instantiationOutput, instantiationPair.second, Lang.NTRIPLES)
val instantiationData = instantiationOutput.toString() val instantiationData = instantiationOutput.toString()
log.info("Ingesting instantiation ${instantiationPair.first}.") log.info("Ingesting instantiation ${instantiationPair.first}.")
transaction.createOrUpdateRdfResource(URI(instantiationPair.first), instantiationData, RdfContentTypes.NTRIPLES) transaction.createOrUpdateRdfResource(
URI(instantiationPair.first),
instantiationData,
RdfContentTypes.NTRIPLES
)
log.info("Ingested instantiation ${instantiationPair.first}.") log.info("Ingested instantiation ${instantiationPair.first}.")
} }
} }
private fun ingestBinaries(sftpLocators: List<Pair<String, String?>>, rdfHandler: RdfHandler, transaction: FedoraTransactionClient) { private fun ingestBinaries(
sftpLocators: List<Pair<String, String?>>,
rdfHandler: RdfHandler,
transaction: FedoraTransactionClient
) {
sftpLocators.forEach { sftpLocators.forEach {
val digitalInstantiationUrl = it.first val digitalInstantiationUrl = it.first
it.second.let { path -> it.second.let { path ->
......
package org.memobase package org.memobase
import java.io.ByteArrayInputStream import ch.memobase.rdf.EBUCORE
import ch.memobase.rdf.RDF
import ch.memobase.rdf.RICO
import org.apache.jena.rdf.model.Model import org.apache.jena.rdf.model.Model
import org.apache.jena.rdf.model.ModelFactory import org.apache.jena.rdf.model.ModelFactory
import org.apache.jena.rdf.model.Statement import org.apache.jena.rdf.model.Statement
...@@ -10,9 +12,7 @@ import org.apache.jena.riot.Lang ...@@ -10,9 +12,7 @@ import org.apache.jena.riot.Lang
import org.apache.jena.riot.RDFDataMgr import org.apache.jena.riot.RDFDataMgr
import org.apache.log4j.LogManager import org.apache.log4j.LogManager
import org.memobase.exceptions.MissingMimeTypeException import org.memobase.exceptions.MissingMimeTypeException
import org.memobase.rdf.EBUCORE import java.io.ByteArrayInputStream
import org.memobase.rdf.RDF
import org.memobase.rdf.RICO
class RdfHandler(data: String, private val externalBaseUrl: String) { class RdfHandler(data: String, private val externalBaseUrl: String) {
private val log = LogManager.getLogger("IngestRdfHandler") private val log = LogManager.getLogger("IngestRdfHandler")
......
...@@ -18,18 +18,19 @@ ...@@ -18,18 +18,19 @@
package org.memobase package org.memobase
import java.io.Closeable import ch.memobase.exceptions.SftpClientException
import java.util.Properties import ch.memobase.reporting.ReportStatus
import ch.memobase.settings.SettingsLoader
import ch.memobase.sftp.SftpClient
import org.apache.kafka.clients.consumer.ConsumerRecord import org.apache.kafka.clients.consumer.ConsumerRecord
import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.LogManager
import org.apache.logging.log4j.Logger import org.apache.logging.log4j.Logger
import org.fcrepo.client.FcrepoOperationFailedException import org.fcrepo.client.FcrepoOperationFailedException
import org.memobase.exceptions.MissingMimeTypeException import org.memobase.exceptions.MissingMimeTypeException
import org.memobase.exceptions.SftpClientException
import org.memobase.fedora.FedoraClient import org.memobase.fedora.FedoraClient
import org.memobase.fedora.FedoraClientImpl import org.memobase.fedora.FedoraClientImpl
import org.memobase.settings.SettingsLoader import java.io.Closeable
import org.memobase.sftp.SftpClient import java.util.Properties
class Service(fileName: String = "app.yml") : Closeable { class Service(fileName: String = "app.yml") : Closeable {
...@@ -108,10 +109,10 @@ class Service(fileName: String = "app.yml") : Closeable { ...@@ -108,10 +109,10 @@ class Service(fileName: String = "app.yml") : Closeable {
val value = record.value() val value = record.value()
if (key == null) if (key == null)
return Report("NoKey", ReportStatus.failure, "The key in message is null.") return Report("NoKey", ReportStatus.fatal, "The key in message is null.")
if (value == null) if (value == null)
return Report(key, ReportStatus.failure, "The value in message is null.") return Report(key, ReportStatus.fatal, "The value in message is null.")
return try { return try {
simpleIngester.ingest(key, value) simpleIngester.ingest(key, value)
...@@ -121,7 +122,7 @@ class Service(fileName: String = "app.yml") : Closeable { ...@@ -121,7 +122,7 @@ class Service(fileName: String = "app.yml") : Closeable {
} catch (ex: FcrepoOperationFailedException) { } catch (ex: FcrepoOperationFailedException) {
log.error(ex.localizedMessage) log.error(ex.localizedMessage)
Report( Report(
key, ReportStatus.failure, ex.localizedMessage key, ReportStatus.fatal, ex.localizedMessage
) )
} }
} }
...@@ -146,28 +147,28 @@ class Service(fileName: String = "app.yml") : Closeable { ...@@ -146,28 +147,28 @@ class Service(fileName: String = "app.yml") : Closeable {
log.error("${ex.javaClass.canonicalName}:: ${ex.localizedMessage}", ex) log.error("${ex.javaClass.canonicalName}:: ${ex.localizedMessage}", ex)
Report( Report(
id = record.key(), id = record.key(),
status = ReportStatus.failure, status = ReportStatus.fatal,
message = "Fedora Exception: ${ex.localizedMessage}" message = "Fedora Exception: ${ex.localizedMessage}"
) )
} catch (ex: MissingMimeTypeException) { } catch (ex: MissingMimeTypeException) {
log.error("${ex.javaClass.canonicalName}:: ${ex.localizedMessage}", ex) log.error("${ex.javaClass.canonicalName}:: ${ex.localizedMessage}", ex)
Report( Report(
id = record.key(), id = record.key(),
status = ReportStatus.failure, status = ReportStatus.fatal,
message = "Missing MimeType Exception: ${ex.localizedMessage}" message = "Missing MimeType Exception: ${ex.localizedMessage}"
) )
} catch (ex: SftpClientException) { } catch (ex: SftpClientException) {
log.error("${ex.javaClass.canonicalName}: ${ex.localizedMessage}", ex) log.error("${ex.javaClass.canonicalName}: ${ex.localizedMessage}", ex)
Report( Report(
id = record.key(), id = record.key(),
status = ReportStatus.failure, status = ReportStatus.fatal,
message = "SFTP Exception: ${ex.localizedMessage}" message = "SFTP Exception: ${ex.localizedMessage}"
) )
} catch (ex: Exception) { } catch (ex: Exception) {
log.error("${ex.javaClass.canonicalName}: ${ex.localizedMessage}", ex) log.error("${ex.javaClass.canonicalName}: ${ex.localizedMessage}", ex)
Report( Report(
id = record.key(), id = record.key(),
status = ReportStatus.failure, status = ReportStatus.fatal,
message = "Unknown Exception: ${ex.localizedMessage}" message = "Unknown Exception: ${ex.localizedMessage}"
) )
} }
......
...@@ -18,25 +18,8 @@ ...@@ -18,25 +18,8 @@
package org.memobase package org.memobase
object ReportStatus {
const val success = "SUCCESS"
const val failure = "FAILURE"
}
object ReportMessages { object ReportMessages {
fun processFailure(failures: Int, total: Int): String {
return "Failed to ingest $failures of $total resources."
}
fun processSuccess(total: Int): String {
return "Successfully ingested $total resources."
}
fun ingestedRecord(id: String): String { fun ingestedRecord(id: String): String {
return "Ingested resource $id." return "Ingested resource $id."
} }
fun ingestFailed(id: String): String {
return "Failed to ingest resource $id."
}
} }
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment