Due to a scheduled upgrade to version 14.10, GitLab will be unavailabe on Monday 30.05., from 19:00 until 20:00.

Service.kt 2.87 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
/*
 * text-file-validation
 * Copyright (C) 2020 Memoriav
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */

package org.memobase

import java.io.File
Jonas Waeber's avatar
Jonas Waeber committed
22
import org.apache.logging.log4j.LogManager
23
24
25
26
import org.memobase.settings.SettingsLoader
import org.memobase.sftp.SftpClient

class Service(fileName: String = "app.yml") {
Jonas Waeber's avatar
Jonas Waeber committed
27
    val log = LogManager.getLogger("TextFileValidationService")
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43

    val settings =
        SettingsLoader(
            listOf("directory"),
            fileName,
            useProducerConfig = true,
            readSftpSettings = true
        )

    private val producer = Producer(settings.kafkaProducerSettings, settings.outputTopic)

    private val sftpClient = SftpClient(settings.sftpSettings)

    fun run() {
        producer.use { producer ->
            sftpClient.use { sftp ->
Jonas Waeber's avatar
Jonas Waeber committed
44
                log.info("Connected to SFTP & Kafka.")
45
46
47
48
                val validator = FileValidation(sftp)
                val files = sftp.listFiles(settings.appSettings.getProperty("directory")).map { File(it) }
                val reports = mutableListOf<Report>()
                for (file in files) {
Jonas Waeber's avatar
Jonas Waeber committed
49
                    log.info("Validate file ${file.absolutePath}.")
50
51
52
53
54
55
56
                    val validationResult = validator.validate(file)
                    producer.sendMessage(validationResult.second.id, validationResult.first)
                    producer.sendReport(validationResult.second)
                    reports.add(validationResult.second)
                }
                val failures = reports.count { report -> report.status == "FAILURE" }
                if (failures > 0) {
Jonas Waeber's avatar
Jonas Waeber committed
57
                    log.info("Validation ended with $failures failures!")
58
59
60
61
62
                    producer.sendJobReport(
                        Report(settings.id, status = "FAILURE", message = "Failed to validate $failures of ${reports.size} files."),
                        settings.processReportTopic
                    )
                } else {
Jonas Waeber's avatar
Jonas Waeber committed
63
                    log.info("Validation was successful!")
64
65
66
67
68
69
70
71
72
                    producer.sendJobReport(
                        Report(settings.id, status = "SUCCESS", message = "Successfully validated ${reports.size} files."),
                        settings.processReportTopic
                    )
                }
            }
        }
    }
}