/* * text-file-validation * Copyright (C) 2020 Memoriav * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Affero General Public License for more details. * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ package org.memobase import java.io.File import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.Logger import org.memobase.settings.SettingsLoader import org.memobase.sftp.SftpClient class Service(fileName: String = "app.yml") { private val log: Logger = LogManager.getLogger("TextFileValidationService") val settings = SettingsLoader( listOf("directory"), fileName, useProducerConfig = true, readSftpSettings = true ) private val producer = Producer(settings.kafkaProducerSettings, settings.outputTopic) private val sftpClient = SftpClient(settings.sftpSettings) fun run() { producer.use { producer -> sftpClient.use { sftp -> log.info("Connected to SFTP & Kafka.") val validator = FileValidation() val files = sftp.listFiles(settings.appSettings.getProperty("directory")).map { File(it) } val reports = mutableListOf() for (file in files) { log.info("Validate file ${file.absolutePath}.") val format = validator.validateExtension(file) val validationResult = validator.validate(sftp.open(file), format, file) producer.sendMessage(validationResult.second.id, validationResult.first) producer.sendReport(validationResult.second) reports.add(validationResult.second) } val failures = reports.count { report -> report.status == ReportStatus.failure } if (failures > 0) { log.warn("Validation ended with $failures failures!") producer.sendJobReport( Report("text-file-validation", status = ReportStatus.failure, message = ReportMessages.processFailure(failures, reports.size)), settings.processReportTopic ) } else { log.info("Validation was successful!") producer.sendJobReport( Report("text-file-validation", status = ReportStatus.success, message = ReportMessages.processSuccess(reports.size)), settings.processReportTopic ) } } } } }