Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
memoriav
Memobase 2020
services
Import Process
Fedora Ingest Service
Commits
b5ca4b43
Commit
b5ca4b43
authored
Oct 23, 2020
by
Jonas Waeber
Browse files
Make variables for constants in code
parent
cdd62f33
Changes
2
Hide whitespace changes
Inline
Side-by-side
src/main/kotlin/RdfHandler.kt
View file @
b5ca4b43
...
...
@@ -6,18 +6,20 @@ import org.apache.jena.rdf.model.ModelFactory
import
org.apache.jena.rdf.model.Statement
import
org.apache.jena.rdf.model.impl.SelectorImpl
import
org.apache.jena.rdf.model.impl.StatementImpl
import
org.apache.jena.riot.Lang
import
org.apache.jena.riot.RDFDataMgr
import
org.apache.log4j.LogManager
import
org.memobase.rdf.EBUCORE
import
org.memobase.rdf.RDF
import
org.memobase.rdf.RICO
import
java.io.StringReader
class
RdfHandler
(
data
:
String
,
externalBaseUrl
:
String
)
{
private
val
log
=
LogManager
.
getLogger
(
"RdfHandler"
)
class
RdfHandler
(
data
:
String
,
private
val
externalBaseUrl
:
String
)
{
private
val
log
=
LogManager
.
getLogger
(
"
Ingest
RdfHandler"
)
private
val
model
=
ModelFactory
.
createDefaultModel
()
private
val
externalBaseUrl
=
externalBaseUrl
init
{
model
.
read
(
ByteArrayInputStream
(
data
.
toByteArray
())
,
""
,
"
NTRIPLES
"
)
RDFDataMgr
.
read
(
model
,
ByteArrayInputStream
(
data
.
toByteArray
())
,
Lang
.
NTRIPLES
)
}
fun
getRecord
():
Pair
<
String
,
Model
>
{
...
...
@@ -45,7 +47,7 @@ class RdfHandler(data: String, externalBaseUrl: String) {
}
private
fun
isInternalNonBinaryResource
(
uri
:
String
):
Boolean
{
return
uri
.
startsWith
(
externalBaseUrl
)
&&
!
uri
.
endsWith
(
"/
binary
"
)
return
uri
.
startsWith
(
externalBaseUrl
)
&&
!
uri
.
endsWith
(
"/
${Service.BINARY_FILE_URI_PATH}
"
)
}
fun
getReferencedNonBinaryResources
():
List
<
String
>
{
...
...
@@ -79,9 +81,10 @@ class RdfHandler(data: String, externalBaseUrl: String) {
private
fun
replaceSftpLocator
(
uri
:
String
,
model
:
Model
)
{
var
newStatement
:
Statement
?
=
null
val
string
:
String
?
=
null
val
replacementLocator
=
"$uri/${Service.BINARY_FILE_URI_PATH}"
val
removedStatement
=
model
.
listStatements
(
SelectorImpl
(
null
,
EBUCORE
.
locator
,
string
)).
mapWith
{
if
(
it
.
`object`
.
asLiteral
().
string
.
startsWith
(
"sftp:"
))
{
newStatement
=
StatementImpl
(
it
.
subject
,
it
.
predicate
,
model
.
createLiteral
(
"$uri/binary"
))
if
(
it
.
`object`
.
asLiteral
().
string
.
startsWith
(
Service
.
SFTP_PREFIX
))
{
newStatement
=
StatementImpl
(
it
.
subject
,
it
.
predicate
,
model
.
createLiteral
(
replacementLocator
))
}
it
}.
nextOptional
()
...
...
@@ -91,17 +94,16 @@ class RdfHandler(data: String, externalBaseUrl: String) {
model
.
add
(
newStatement
)
}
}
log
.
error
(
"Removed statement $removedStatement."
)
log
.
error
(
"Added statement $newStatement."
)
log
.
info
(
"Replaced sftp locator with fedora url: $replacementLocator."
)
}
fun
getSftpLocators
():
List
<
Pair
<
String
,
String
?
>>
{
return
model
.
listSubjectsWithProperty
(
RICO
.
type
,
"digitalObject"
).
filterKeep
{
return
model
.
listSubjectsWithProperty
(
RICO
.
type
,
Service
.
DIGITAL_OBJECT_TYPE
).
filterKeep
{
it
.
hasProperty
(
EBUCORE
.
locator
)
}.
mapWith
{
val
url
=
it
.
getProperty
(
EBUCORE
.
locator
).
`object`
.
asLiteral
().
string
if
(
url
.
startsWith
(
"sftp:"
))
{
Pair
(
it
.
uri
,
url
.
replace
(
"sftp:"
,
""
))
if
(
url
.
startsWith
(
Service
.
SFTP_PREFIX
))
{
Pair
(
it
.
uri
,
url
.
replace
(
Service
.
SFTP_PREFIX
,
""
))
}
else
{
Pair
(
it
.
uri
,
null
)
}
...
...
@@ -109,7 +111,7 @@ class RdfHandler(data: String, externalBaseUrl: String) {
}
fun
getMimeType
():
List
<
Pair
<
String
,
String
?
>>
{
return
model
.
listSubjectsWithProperty
(
RICO
.
type
,
"digitalObject"
).
filterKeep
{
return
model
.
listSubjectsWithProperty
(
RICO
.
type
,
Service
.
DIGITAL_OBJECT_TYPE
).
filterKeep
{
it
.
hasProperty
(
EBUCORE
.
hasMimeType
)
}.
mapWith
{
Pair
(
it
.
uri
,
it
.
getProperty
(
EBUCORE
.
hasMimeType
).
`object`
.
asLiteral
().
string
)
...
...
src/main/kotlin/Service.kt
View file @
b5ca4b43
...
...
@@ -31,10 +31,14 @@ import org.memobase.settings.SettingsLoader
class
Service
(
fileName
:
String
=
"app.yml"
)
:
Closeable
{
companion
object
{
const
val
DIGITAL_OBJECT_TYPE
=
"digitalObject"
const
val
FEDORA_PROPERTIES_PREFIX
=
"fedora"
const
val
CONSUMER_MAX_POLL_RECORDS
=
"10"
const
val
CONSUMER_MAX_INTERVAL_MS
=
"3600000"
// 3600000ms = 1h
const
val
BINARY_FILE_URI_PATH
=
"binary"
const
val
SFTP_PREFIX
=
"sftp:"
fun
createFedoraClient
(
appSettings
:
Properties
):
FedoraClient
{
return
FedoraClientImpl
.
builder
()
.
properties
(
appSettings
,
FEDORA_PROPERTIES_PREFIX
)
...
...
@@ -42,7 +46,7 @@ class Service(fileName: String = "app.yml") : Closeable {
}
}
val
settings
=
SettingsLoader
(
private
val
settings
=
SettingsLoader
(
listOf
(
"$FEDORA_PROPERTIES_PREFIX.internalBaseUrl"
,
"$FEDORA_PROPERTIES_PREFIX.externalBaseUrl"
,
...
...
@@ -82,14 +86,6 @@ class Service(fileName: String = "app.yml") : Closeable {
}
private
fun
processRecord
(
record
:
ConsumerRecord
<
String
,
String
>):
Report
{
if
(
record
.
value
().
contains
(
"ERROR"
))
{
return
Report
(
id
=
record
.
key
(),
status
=
ReportStatus
.
failure
,
message
=
ReportMessages
.
ingestFailed
(
record
.
key
())
)
}
val
ingester
=
Ingester
(
settings
.
sftpSettings
,
createFedoraClient
(
settings
.
appSettings
),
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment