Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
memoriav
Memobase 2020
services
Import Process
Fedora Ingest Service
Commits
9f925305
Commit
9f925305
authored
Dec 07, 2020
by
Jonas Waeber
Browse files
Update utility dependency to 2.0.2.
Failures are now fatal reports.
parent
61db65ac
Changes
5
Hide whitespace changes
Inline
Side-by-side
build.gradle
View file @
9f925305
...
...
@@ -45,7 +45,7 @@ dependencies {
implementation
'org.apache.jena:apache-jena:3.14.0'
,
excludeSlf4J
implementation
'org.memobase:memobase-service-utilities:
1.12
.2'
,
excludeSlf4J
implementation
'org.memobase:memobase-service-utilities:
2.0
.2'
,
excludeSlf4J
implementation
'org.memobase:fedora-client:0.6.2'
,
excludeSlf4J
// KOTLIN IMPORTS
...
...
src/main/kotlin/Ingester.kt
View file @
9f925305
...
...
@@ -17,9 +17,7 @@
*/
package
org.memobase
import
java.io.File
import
java.io.StringWriter
import
java.net.URI
import
ch.memobase.sftp.SftpClient
import
org.apache.jena.rdf.model.Model
import
org.apache.jena.riot.Lang
import
org.apache.jena.riot.RDFDataMgr
...
...
@@ -28,7 +26,9 @@ import org.fcrepo.client.FcrepoOperationFailedException
import
org.memobase.fedora.FedoraClient
import
org.memobase.fedora.FedoraTransactionClient
import
org.memobase.fedora.RdfContentTypes
import
org.memobase.sftp.SftpClient
import
java.io.File
import
java.io.StringWriter
import
java.net.URI
class
Ingester
(
private
val
sftpClient
:
SftpClient
,
...
...
@@ -76,12 +76,20 @@ class Ingester(
RDFDataMgr
.
write
(
instantiationOutput
,
instantiationPair
.
second
,
Lang
.
NTRIPLES
)
val
instantiationData
=
instantiationOutput
.
toString
()
log
.
info
(
"Ingesting instantiation ${instantiationPair.first}."
)
transaction
.
createOrUpdateRdfResource
(
URI
(
instantiationPair
.
first
),
instantiationData
,
RdfContentTypes
.
NTRIPLES
)
transaction
.
createOrUpdateRdfResource
(
URI
(
instantiationPair
.
first
),
instantiationData
,
RdfContentTypes
.
NTRIPLES
)
log
.
info
(
"Ingested instantiation ${instantiationPair.first}."
)
}
}
private
fun
ingestBinaries
(
sftpLocators
:
List
<
Pair
<
String
,
String
?
>>,
rdfHandler
:
RdfHandler
,
transaction
:
FedoraTransactionClient
)
{
private
fun
ingestBinaries
(
sftpLocators
:
List
<
Pair
<
String
,
String
?
>>,
rdfHandler
:
RdfHandler
,
transaction
:
FedoraTransactionClient
)
{
sftpLocators
.
forEach
{
val
digitalInstantiationUrl
=
it
.
first
it
.
second
.
let
{
path
->
...
...
src/main/kotlin/RdfHandler.kt
View file @
9f925305
package
org.memobase
import
java.io.ByteArrayInputStream
import
ch.memobase.rdf.EBUCORE
import
ch.memobase.rdf.RDF
import
ch.memobase.rdf.RICO
import
org.apache.jena.rdf.model.Model
import
org.apache.jena.rdf.model.ModelFactory
import
org.apache.jena.rdf.model.Statement
...
...
@@ -10,9 +12,7 @@ import org.apache.jena.riot.Lang
import
org.apache.jena.riot.RDFDataMgr
import
org.apache.log4j.LogManager
import
org.memobase.exceptions.MissingMimeTypeException
import
org.memobase.rdf.EBUCORE
import
org.memobase.rdf.RDF
import
org.memobase.rdf.RICO
import
java.io.ByteArrayInputStream
class
RdfHandler
(
data
:
String
,
private
val
externalBaseUrl
:
String
)
{
private
val
log
=
LogManager
.
getLogger
(
"IngestRdfHandler"
)
...
...
src/main/kotlin/Service.kt
View file @
9f925305
...
...
@@ -18,18 +18,19 @@
package
org.memobase
import
java.io.Closeable
import
java.util.Properties
import
ch.memobase.exceptions.SftpClientException
import
ch.memobase.reporting.ReportStatus
import
ch.memobase.settings.SettingsLoader
import
ch.memobase.sftp.SftpClient
import
org.apache.kafka.clients.consumer.ConsumerRecord
import
org.apache.logging.log4j.LogManager
import
org.apache.logging.log4j.Logger
import
org.fcrepo.client.FcrepoOperationFailedException
import
org.memobase.exceptions.MissingMimeTypeException
import
org.memobase.exceptions.SftpClientException
import
org.memobase.fedora.FedoraClient
import
org.memobase.fedora.FedoraClientImpl
import
org.memobase.settings.SettingsLoader
import
org.memobase.sftp.SftpClient
import
java.io.Closeable
import
java.util.Properties
class
Service
(
fileName
:
String
=
"app.yml"
)
:
Closeable
{
...
...
@@ -108,10 +109,10 @@ class Service(fileName: String = "app.yml") : Closeable {
val
value
=
record
.
value
()
if
(
key
==
null
)
return
Report
(
"NoKey"
,
ReportStatus
.
fa
ilure
,
"The key in message is null."
)
return
Report
(
"NoKey"
,
ReportStatus
.
fa
tal
,
"The key in message is null."
)
if
(
value
==
null
)
return
Report
(
key
,
ReportStatus
.
fa
ilure
,
"The value in message is null."
)
return
Report
(
key
,
ReportStatus
.
fa
tal
,
"The value in message is null."
)
return
try
{
simpleIngester
.
ingest
(
key
,
value
)
...
...
@@ -121,7 +122,7 @@ class Service(fileName: String = "app.yml") : Closeable {
}
catch
(
ex
:
FcrepoOperationFailedException
)
{
log
.
error
(
ex
.
localizedMessage
)
Report
(
key
,
ReportStatus
.
fa
ilure
,
ex
.
localizedMessage
key
,
ReportStatus
.
fa
tal
,
ex
.
localizedMessage
)
}
}
...
...
@@ -146,28 +147,28 @@ class Service(fileName: String = "app.yml") : Closeable {
log
.
error
(
"${ex.javaClass.canonicalName}:: ${ex.localizedMessage}"
,
ex
)
Report
(
id
=
record
.
key
(),
status
=
ReportStatus
.
fa
ilure
,
status
=
ReportStatus
.
fa
tal
,
message
=
"Fedora Exception: ${ex.localizedMessage}"
)
}
catch
(
ex
:
MissingMimeTypeException
)
{
log
.
error
(
"${ex.javaClass.canonicalName}:: ${ex.localizedMessage}"
,
ex
)
Report
(
id
=
record
.
key
(),
status
=
ReportStatus
.
fa
ilure
,
status
=
ReportStatus
.
fa
tal
,
message
=
"Missing MimeType Exception: ${ex.localizedMessage}"
)
}
catch
(
ex
:
SftpClientException
)
{
log
.
error
(
"${ex.javaClass.canonicalName}: ${ex.localizedMessage}"
,
ex
)
Report
(
id
=
record
.
key
(),
status
=
ReportStatus
.
fa
ilure
,
status
=
ReportStatus
.
fa
tal
,
message
=
"SFTP Exception: ${ex.localizedMessage}"
)
}
catch
(
ex
:
Exception
)
{
log
.
error
(
"${ex.javaClass.canonicalName}: ${ex.localizedMessage}"
,
ex
)
Report
(
id
=
record
.
key
(),
status
=
ReportStatus
.
fa
ilure
,
status
=
ReportStatus
.
fa
tal
,
message
=
"Unknown Exception: ${ex.localizedMessage}"
)
}
...
...
src/main/kotlin/Values.kt
View file @
9f925305
...
...
@@ -18,25 +18,8 @@
package
org.memobase
object
ReportStatus
{
const
val
success
=
"SUCCESS"
const
val
failure
=
"FAILURE"
}
object
ReportMessages
{
fun
processFailure
(
failures
:
Int
,
total
:
Int
):
String
{
return
"Failed to ingest $failures of $total resources."
}
fun
processSuccess
(
total
:
Int
):
String
{
return
"Successfully ingested $total resources."
}
fun
ingestedRecord
(
id
:
String
):
String
{
return
"Ingested resource $id."
}
fun
ingestFailed
(
id
:
String
):
String
{
return
"Failed to ingest resource $id."
}
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment