Commit dbf8fa11 authored by Jonas Waeber's avatar Jonas Waeber

Refactored process reporting

parent 3c1e2574
Pipeline #12648 passed with stages
in 7 minutes and 5 seconds
## Text File validation Service
[![pipeline status](https://gitlab.switch.ch/memoriav/memobase-2020/services/import-process/text-file-validation/badges/master/pipeline.svg)](https://gitlab.switch.ch/memoriav/memobase-2020/services/import-process/text-file-validation/-/commits/master)
Checks the files present in the `app.directory` folder on the sFTP server.
[Confluence Doku](https://memobase.atlassian.net/wiki/spaces/TBAS/pages/29196525/Service+Text+File+Validation)
......
......@@ -18,7 +18,13 @@
package org.memobase
import com.beust.klaxon.Klaxon
data class Message(
val format: String,
val path: String
)
) {
fun toJson(): String {
return Klaxon().toJsonString(this)
}
}
/*
* xml-data-transform
* Copyright (C) 2020 Memoriav
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.memobase
import com.beust.klaxon.Klaxon
data class ProcessReport(
val id: String,
val status: String,
val total: Int,
val successes: Int,
val failures: Int
) {
fun toJson(): String {
return Klaxon().toJsonString(this)
}
}
......@@ -17,7 +17,6 @@
*/
package org.memobase
import com.beust.klaxon.Klaxon
import java.io.Closeable
import java.util.Properties
import org.apache.kafka.clients.producer.KafkaProducer
......@@ -28,15 +27,15 @@ class Producer(props: Properties, private val topic: String) : Closeable {
private val reportingTopic = "$topic-reporting"
fun sendMessage(key: String, message: Message) {
instance.send(ProducerRecord(topic, key, Klaxon().toJsonString(message)))
instance.send(ProducerRecord(topic, key, message.toJson()))
}
fun sendReport(report: Report) {
instance.send(ProducerRecord(reportingTopic, report.id, Klaxon().toJsonString(report)))
instance.send(ProducerRecord(reportingTopic, report.id, report.toJson()))
}
fun sendJobReport(report: Report, topic: String) {
instance.send(ProducerRecord(topic, report.id, Klaxon().toJsonString(report)))
fun sendJobReport(report: ProcessReport, topic: String) {
instance.send(ProducerRecord(topic, report.id, report.toJson()))
}
override fun close() {
......
......@@ -18,8 +18,14 @@
package org.memobase
import com.beust.klaxon.Klaxon
data class Report(
val id: String,
val status: String,
val message: String
)
) {
fun toJson(): String {
return Klaxon().toJsonString(this)
}
}
......@@ -19,6 +19,7 @@
package org.memobase
import java.io.File
import kotlin.system.exitProcess
import org.apache.logging.log4j.LogManager
import org.apache.logging.log4j.Logger
import org.memobase.exceptions.SftpClientException
......@@ -37,18 +38,38 @@ class Service(fileName: String = "app.yml") {
)
private val producer = Producer(settings.kafkaProducerSettings, settings.outputTopic)
private val directory = settings.appSettings.getProperty("directory")
private val validator = FileValidation()
private val sftpClient = SftpClient(settings.sftpSettings)
private var totalCount = 0
fun run() {
producer.use { producer ->
sftpClient.use { sftp ->
log.info("Connected to SFTP & Kafka.")
val files = try {
val fileList = sftp.listFiles(directory).map { File(it) }
totalCount = fileList.size
fileList
} catch (ex: SftpClientException) {
ex.printStackTrace()
log.error("SFTP Exception: Could not compile the file list on sftp server in directory: $directory.")
val report = ProcessReport(
"text-file-validation",
status = ReportStatus.failure,
total = 0,
successes = 0,
failures = 0
)
producer.sendJobReport(report, settings.processReportTopic)
exitProcess(1)
}
log.info("Retrieved file list from sftp server at path: $directory")
val reports = mutableListOf<Report>()
try {
val validator = FileValidation()
val files = sftp.listFiles(settings.appSettings.getProperty("directory")).map { File(it) }
val reports = mutableListOf<Report>()
log.info("There are a total of ${files.size} files to validate ")
log.info("There are a total of ${files.size} files to validate.")
for (file in files) {
log.info("Validate file $file.")
val format = validator.validateExtension(file)
......@@ -78,20 +99,24 @@ class Service(fileName: String = "app.yml") {
if (failures > 0) {
log.warn("Validation ended with $failures failures!")
producer.sendJobReport(
Report(
ProcessReport(
"text-file-validation",
status = ReportStatus.failure,
message = ReportMessages.processFailure(failures, reports.size)
total = totalCount,
failures = failures,
successes = totalCount - failures
),
settings.processReportTopic
)
} else {
log.info("Validation was successful!")
producer.sendJobReport(
Report(
ProcessReport(
"text-file-validation",
status = ReportStatus.success,
message = ReportMessages.processSuccess(reports.size)
total = totalCount,
successes = reports.size,
failures = 0
),
settings.processReportTopic
)
......@@ -99,20 +124,24 @@ class Service(fileName: String = "app.yml") {
} catch (ex: SftpClientException) {
ex.printStackTrace()
log.error("SFTP Exception: ${ex.localizedMessage}.")
val report = Report(
val report = ProcessReport(
"text-file-validation",
status = ReportStatus.failure,
message = "SFTP Exception: ${ex.localizedMessage}."
total = totalCount,
failures = reports.count { report -> report.status == ReportStatus.failure },
successes = reports.count { report -> report.status == ReportStatus.success }
)
producer.sendJobReport(report, settings.processReportTopic)
} catch (ex: Exception) {
ex.printStackTrace()
log.error(ex.javaClass.canonicalName + ": " + ex.localizedMessage)
producer.sendJobReport(
Report(
ProcessReport(
"text-file-validation",
status = ReportStatus.failure,
message = "Unknown Exception: ${ex.javaClass.canonicalName}: ${ex.localizedMessage}."
total = totalCount,
failures = reports.count { report -> report.status == ReportStatus.failure },
successes = reports.count { report -> report.status == ReportStatus.success }
),
settings.processReportTopic
)
......
......@@ -22,6 +22,6 @@ data class TestParams(
val configFile: String,
val expectedKey: String,
val expectedValue: String,
val expectedReportValue: String,
val expectedProcessReport: Report
val expectedReportValue: Report,
val expectedProcessReport: ProcessReport
)
This diff is collapsed.
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment