Ingester.kt 4.29 KB
Newer Older
Jonas Waeber's avatar
Jonas Waeber committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
/*
 * fedora-ingest-service
 * Copyright (C) 2020 Memoriav
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
Thomas Bernhart's avatar
Thomas Bernhart committed
18
19
package org.memobase

Thomas Bernhart's avatar
Thomas Bernhart committed
20
21
22
import java.io.File
import java.io.StringWriter
import java.net.URI
23
import org.apache.jena.rdf.model.Model
Jonas Waeber's avatar
Jonas Waeber committed
24
25
import org.apache.jena.riot.Lang
import org.apache.jena.riot.RDFDataMgr
Thomas Bernhart's avatar
Thomas Bernhart committed
26
27
28
import org.apache.logging.log4j.LogManager
import org.fcrepo.client.FcrepoOperationFailedException
import org.memobase.fedora.FedoraClient
29
import org.memobase.fedora.FedoraTransactionClient
30
import org.memobase.fedora.RdfContentTypes
Thomas Bernhart's avatar
Thomas Bernhart committed
31
32
import org.memobase.sftp.SftpClient

33
class Ingester(
34
    private val sftpClient: SftpClient,
35
36
    private val fedoraClient: FedoraClient,
    private val externalBaseUrl: String
37
) {
Thomas Bernhart's avatar
Thomas Bernhart committed
38

39
    private val log = LogManager.getLogger("FedoraIngester")
Thomas Bernhart's avatar
Thomas Bernhart committed
40
41

    @Throws(FcrepoOperationFailedException::class)
42
    fun ingest(id: String, content: String) {
Jonas Waeber's avatar
Jonas Waeber committed
43
        log.info("Begin ingest of message with id $id.")
44
        val rdfHandler = RdfHandler(content, externalBaseUrl)
45
46
        val recordOutput = StringWriter()
        val recordPair = rdfHandler.getRecord()
Jonas Waeber's avatar
Jonas Waeber committed
47
        RDFDataMgr.write(recordOutput, recordPair.second, Lang.NTRIPLES)
48
        val data = recordOutput.toString()
49

50
51
52
53
54
55
56
57
58
59
        fedoraClient.startTransaction().use { transaction ->
            log.info("Ingesting record ${recordPair.first}.")

            // create placeholders referenced resources:
            val nonBinaryResources = rdfHandler.getReferencedNonBinaryResources()
            nonBinaryResources.forEach { resource ->
                log.info("Creating placeholder for resource $resource.")
                transaction.createPlaceholder(URI(resource))
                log.info("Created placeholder for resource $resource.")
            }
60

61
62
63
            // ingest record, instantiations and binaries:
            transaction.createOrUpdateRdfResource(URI(recordPair.first), data, RdfContentTypes.NTRIPLES)
            ingestInstantiations(rdfHandler.getInstantiations(), transaction)
64
            val sftpLocators = rdfHandler.getSftpLocators()
Jonas Waeber's avatar
Jonas Waeber committed
65
            if (sftpLocators.isNotEmpty()) {
66
67
                ingestBinaries(sftpLocators, rdfHandler, transaction)
            }
68
69
70
            transaction.commit()
            log.info("Ingested record ${recordPair.first}.")
        }
71
    }
72

73
    private fun ingestInstantiations(instantiations: List<Pair<String, Model>>, transaction: FedoraTransactionClient) {
74
        instantiations.forEach { instantiationPair ->
75
            val instantiationOutput = StringWriter()
Jonas Waeber's avatar
Jonas Waeber committed
76
            RDFDataMgr.write(instantiationOutput, instantiationPair.second, Lang.NTRIPLES)
77
78
79
80
            val instantiationData = instantiationOutput.toString()
            log.info("Ingesting instantiation ${instantiationPair.first}.")
            transaction.createOrUpdateRdfResource(URI(instantiationPair.first), instantiationData, RdfContentTypes.NTRIPLES)
            log.info("Ingested instantiation ${instantiationPair.first}.")
81
        }
82
    }
83

84
    private fun ingestBinaries(sftpLocators: List<Pair<String, String?>>, rdfHandler: RdfHandler, transaction: FedoraTransactionClient) {
85
86
87
88
89
90
91
92
93
        sftpLocators.forEach {
            val digitalInstantiationUrl = it.first
            it.second.let { path ->
                if (path != null) {
                    sftpClient.open(File(path)).use { stream ->
                        val binaryUri = "$digitalInstantiationUrl/${Service.BINARY_FILE_URI_PATH}"
                        val mimeType = rdfHandler.getMimeType(digitalInstantiationUrl)
                        log.info("Ingesting binary $binaryUri with mime type $mimeType.")
                        transaction.createOrUpdateBinaryResource(URI(binaryUri), stream, mimeType)
94
95
96
97
                    }
                }
            }
        }
Thomas Bernhart's avatar
Thomas Bernhart committed
98
    }
99
}