Ingester.kt 4.5 KB
Newer Older
Jonas Waeber's avatar
Jonas Waeber committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
/*
 * fedora-ingest-service
 * Copyright (C) 2020 Memoriav
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
Thomas Bernhart's avatar
Thomas Bernhart committed
18
19
package org.memobase

Thomas Bernhart's avatar
Thomas Bernhart committed
20
21
22
import java.io.File
import java.io.StringWriter
import java.net.URI
23
import org.apache.jena.rdf.model.Model
Jonas Waeber's avatar
Jonas Waeber committed
24
25
import org.apache.jena.riot.Lang
import org.apache.jena.riot.RDFDataMgr
Thomas Bernhart's avatar
Thomas Bernhart committed
26
27
28
import org.apache.logging.log4j.LogManager
import org.fcrepo.client.FcrepoOperationFailedException
import org.memobase.fedora.FedoraClient
29
import org.memobase.fedora.FedoraTransactionClient
30
import org.memobase.fedora.RdfContentTypes
31
import org.memobase.settings.SftpSettings
Thomas Bernhart's avatar
Thomas Bernhart committed
32
33
import org.memobase.sftp.SftpClient

34
class Ingester(
35
    private val sftpSettings: SftpSettings,
36
37
    private val fedoraClient: FedoraClient,
    private val externalBaseUrl: String
38
) {
Thomas Bernhart's avatar
Thomas Bernhart committed
39

40
    private val log = LogManager.getLogger("FedoraIngester")
Thomas Bernhart's avatar
Thomas Bernhart committed
41
42

    @Throws(FcrepoOperationFailedException::class)
43
    fun ingest(id: String, content: String) {
Jonas Waeber's avatar
Jonas Waeber committed
44
        log.info("Begin ingest of message with id $id.")
45
        val rdfHandler = RdfHandler(content, externalBaseUrl)
46
47
        val recordOutput = StringWriter()
        val recordPair = rdfHandler.getRecord()
Jonas Waeber's avatar
Jonas Waeber committed
48
        RDFDataMgr.write(recordOutput, recordPair.second, Lang.NTRIPLES)
49
        val data = recordOutput.toString()
50

51
52
53
54
55
56
57
58
59
60
        fedoraClient.startTransaction().use { transaction ->
            log.info("Ingesting record ${recordPair.first}.")

            // create placeholders referenced resources:
            val nonBinaryResources = rdfHandler.getReferencedNonBinaryResources()
            nonBinaryResources.forEach { resource ->
                log.info("Creating placeholder for resource $resource.")
                transaction.createPlaceholder(URI(resource))
                log.info("Created placeholder for resource $resource.")
            }
61

62
63
64
            // ingest record, instantiations and binaries:
            transaction.createOrUpdateRdfResource(URI(recordPair.first), data, RdfContentTypes.NTRIPLES)
            ingestInstantiations(rdfHandler.getInstantiations(), transaction)
65
            val sftpLocators = rdfHandler.getSftpLocators()
Jonas Waeber's avatar
Jonas Waeber committed
66
            if (sftpLocators.isNotEmpty()) {
67
68
                ingestBinaries(sftpLocators, rdfHandler, transaction)
            }
69
70
71
            transaction.commit()
            log.info("Ingested record ${recordPair.first}.")
        }
72
    }
73

74
    private fun ingestInstantiations(instantiations: List<Pair<String, Model>>, transaction: FedoraTransactionClient) {
75
        instantiations.forEach { instantiationPair ->
76
            val instantiationOutput = StringWriter()
Jonas Waeber's avatar
Jonas Waeber committed
77
            RDFDataMgr.write(instantiationOutput, instantiationPair.second, Lang.NTRIPLES)
78
79
80
81
            val instantiationData = instantiationOutput.toString()
            log.info("Ingesting instantiation ${instantiationPair.first}.")
            transaction.createOrUpdateRdfResource(URI(instantiationPair.first), instantiationData, RdfContentTypes.NTRIPLES)
            log.info("Ingested instantiation ${instantiationPair.first}.")
82
        }
83
    }
84

85
    private fun ingestBinaries(sftpLocators: List<Pair<String, String?>>, rdfHandler: RdfHandler, transaction: FedoraTransactionClient) {
86
87
88
89
90
91
92
        SftpClient(sftpSettings).use { sftpClient ->
            log.info("Connected to sFTP server.")
            sftpLocators.forEach {
                val digitalInstantiationUrl = it.first
                it.second.let { path ->
                    if (path != null) {
                        sftpClient.open(File(path)).use { stream ->
93
94
                            val binaryUri = "$digitalInstantiationUrl/${Service.BINARY_FILE_URI_PATH}"
                            val mimeType = rdfHandler.getMimeType(digitalInstantiationUrl)
95
96
                            log.info("Ingesting binary $binaryUri with mime type $mimeType.")
                            transaction.createOrUpdateBinaryResource(URI(binaryUri), stream, mimeType)
97
                        }
98
99
100
101
                    }
                }
            }
        }
Thomas Bernhart's avatar
Thomas Bernhart committed
102
    }
103
}