Commit aa52ead2 authored by Thomas Bernhart's avatar Thomas Bernhart
Browse files

MEMO-734: Hotfix: don't use transactions during ingest

The fedora client does not correctly handle URLs in RDF literals.
This is a hotfix that temporarily disables the usage of transactions
in the fedora-ingest-service.
parent 8e247802
Pipeline #14347 passed with stages
in 5 minutes and 27 seconds
......@@ -27,7 +27,6 @@ import org.apache.kafka.clients.consumer.ConsumerRecords
import org.apache.logging.log4j.LogManager
import org.fcrepo.client.FcrepoOperationFailedException
import org.memobase.fedora.FedoraClient
import org.memobase.fedora.FedoraTransactionClient
import org.memobase.fedora.RdfContentTypes
import org.memobase.sftp.SftpClient
......@@ -59,24 +58,21 @@ class Ingester(
val data = recordOutput.toString()
fedoraClient.startTransaction().use { transaction ->
log.info("Ingesting record ${recordPair.first}.")
log.info("Ingesting record ${recordPair.first}.")
// create placeholders referenced resources:
val nonBinaryResources = rdfHandler.getReferencedNonBinaryResources()
nonBinaryResources.forEach { resource ->
log.info("Creating placeholder for resource $resource.")
transaction.createPlaceholder(URI(resource))
log.info("Created placeholder for resource $resource.")
}
// ingest record, instantiations and binaries:
transaction.createOrUpdateRdfResource(URI(recordPair.first), data, RdfContentTypes.NTRIPLES)
ingestInstantiations(rdfHandler.getInstantiations(), transaction)
ingestBinaries(rdfHandler.getSftpLocators(), rdfHandler, transaction)
transaction.commit()
log.info("Ingested record ${recordPair.first}.")
// create placeholders referenced resources:
val nonBinaryResources = rdfHandler.getReferencedNonBinaryResources()
nonBinaryResources.forEach { resource ->
log.info("Creating placeholder for resource $resource.")
fedoraClient.createPlaceholder(URI(resource))
log.info("Created placeholder for resource $resource.")
}
// ingest record, instantiations and binaries:
fedoraClient.createOrUpdateRdfResource(URI(recordPair.first), data, RdfContentTypes.NTRIPLES)
ingestInstantiations(rdfHandler.getInstantiations())
ingestBinaries(rdfHandler.getSftpLocators(), rdfHandler)
log.info("Ingested record ${recordPair.first}.")
}
private fun processRecord(record: ConsumerRecord<String, String>): Report {
......@@ -105,14 +101,14 @@ class Ingester(
}
}
private fun ingestInstantiations(instantiations: List<Pair<String, Model>>, transaction: FedoraTransactionClient) {
private fun ingestInstantiations(instantiations: List<Pair<String, Model>>) {
instantiations.forEach { instantiationPair ->
val instantiationOutput = StringWriter()
instantiationPair.second.write(instantiationOutput, "NTRIPLES")
try {
val instantiationData = instantiationOutput.toString()
log.info("Ingesting instantiation ${instantiationPair.first}.")
transaction.createOrUpdateRdfResource(URI(instantiationPair.first), instantiationData, RdfContentTypes.NTRIPLES)
fedoraClient.createOrUpdateRdfResource(URI(instantiationPair.first), instantiationData, RdfContentTypes.NTRIPLES)
log.info("Ingested instantiation ${instantiationPair.first}.")
} catch (ex: FcrepoOperationFailedException) {
log.error("Ingestion of instantiation ${instantiationPair.first} failed: ${ex.localizedMessage}.")
......@@ -121,7 +117,7 @@ class Ingester(
}
}
private fun ingestBinaries(sftpLocators: List<Pair<String, String?>>, rdfHandler: RdfHandler, transaction: FedoraTransactionClient) {
private fun ingestBinaries(sftpLocators: List<Pair<String, String?>>, rdfHandler: RdfHandler) {
sftpLocators.forEach {
val digitalInstantiationUrl = it.first
it.second.let { path ->
......@@ -133,7 +129,7 @@ class Ingester(
mT.first == digitalInstantiationUrl
}.second
log.info("Ingesting binary $binaryUri with mime type $mimeType.")
transaction.createOrUpdateBinaryResource(URI(binaryUri), stream, mimeType)
fedoraClient.createOrUpdateBinaryResource(URI(binaryUri), stream, mimeType)
} catch (ex: FcrepoOperationFailedException) {
log.error("Ingestion of binary $binaryUri failed: ${ex.localizedMessage}.")
throw ex
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment