Refactored process reporting

README.md

 ## Text File validation Service
 
+[![pipeline status](https://gitlab.switch.ch/memoriav/memobase-2020/services/import-process/text-file-validation/badges/master/pipeline.svg)](https://gitlab.switch.ch/memoriav/memobase-2020/services/import-process/text-file-validation/-/commits/master)
+
 Checks the files present in the `app.directory` folder on the sFTP server.
 
 [Confluence Doku](https://memobase.atlassian.net/wiki/spaces/TBAS/pages/29196525/Service+Text+File+Validation)

src/main/kotlin/Message.kt

@@ -18,7 +18,13 @@
 
 package org.memobase
 
+import com.beust.klaxon.Klaxon
+
 data class Message(
     val format: String,
     val path: String
-)
+) {
+    fun toJson(): String {
+        return Klaxon().toJsonString(this)
+    }
+}

src/main/kotlin/ProcessReport.kt

+/*
+ * xml-data-transform
+ * Copyright (C) 2020 Memoriav
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+package org.memobase
+
+import com.beust.klaxon.Klaxon
+
+data class ProcessReport(
+    val id: String,
+    val status: String,
+    val total: Int,
+    val successes: Int,
+    val failures: Int
+) {
+    fun toJson(): String {
+        return Klaxon().toJsonString(this)
+    }
+}

src/main/kotlin/Producer.kt

@@ -17,7 +17,6 @@
  */
 package org.memobase
 
-import com.beust.klaxon.Klaxon
 import java.io.Closeable
 import java.util.Properties
 import org.apache.kafka.clients.producer.KafkaProducer
@@ -28,15 +27,15 @@ class Producer(props: Properties, private val topic: String) : Closeable {
     private val reportingTopic = "$topic-reporting"
 
     fun sendMessage(key: String, message: Message) {
-        instance.send(ProducerRecord(topic, key, Klaxon().toJsonString(message)))
+        instance.send(ProducerRecord(topic, key, message.toJson()))
     }
 
     fun sendReport(report: Report) {
-        instance.send(ProducerRecord(reportingTopic, report.id, Klaxon().toJsonString(report)))
+        instance.send(ProducerRecord(reportingTopic, report.id, report.toJson()))
     }
 
-    fun sendJobReport(report: Report, topic: String) {
-        instance.send(ProducerRecord(topic, report.id, Klaxon().toJsonString(report)))
+    fun sendJobReport(report: ProcessReport, topic: String) {
+        instance.send(ProducerRecord(topic, report.id, report.toJson()))
     }
 
     override fun close() {

src/main/kotlin/Report.kt

@@ -18,8 +18,14 @@
 
 package org.memobase
 
+import com.beust.klaxon.Klaxon
+
 data class Report(
     val id: String,
     val status: String,
     val message: String
-)
+) {
+    fun toJson(): String {
+        return Klaxon().toJsonString(this)
+    }
+}

src/main/kotlin/Service.kt

@@ -19,6 +19,7 @@
 package org.memobase
 
 import java.io.File
+import kotlin.system.exitProcess
 import org.apache.logging.log4j.LogManager
 import org.apache.logging.log4j.Logger
 import org.memobase.exceptions.SftpClientException
@@ -37,18 +38,38 @@ class Service(fileName: String = "app.yml") {
         )
 
     private val producer = Producer(settings.kafkaProducerSettings, settings.outputTopic)
+    private val directory = settings.appSettings.getProperty("directory")
+    private val validator = FileValidation()
 
     private val sftpClient = SftpClient(settings.sftpSettings)
 
+    private var totalCount = 0
+
     fun run() {
         producer.use { producer ->
             sftpClient.use { sftp ->
                 log.info("Connected to SFTP & Kafka.")
+                val files = try {
+                    val fileList = sftp.listFiles(directory).map { File(it) }
+                    totalCount = fileList.size
+                    fileList
+                } catch (ex: SftpClientException) {
+                    ex.printStackTrace()
+                    log.error("SFTP Exception: Could not compile the file list on sftp server in directory: $directory.")
+                    val report = ProcessReport(
+                        "text-file-validation",
+                        status = ReportStatus.failure,
+                        total = 0,
+                        successes = 0,
+                        failures = 0
+                    )
+                    producer.sendJobReport(report, settings.processReportTopic)
+                    exitProcess(1)
+                }
+                log.info("Retrieved file list from sftp server at path: $directory")
+                val reports = mutableListOf<Report>()
                 try {
-                    val validator = FileValidation()
-                    val files = sftp.listFiles(settings.appSettings.getProperty("directory")).map { File(it) }
-                    val reports = mutableListOf<Report>()
-                    log.info("There are a total of ${files.size} files to validate ")
+                    log.info("There are a total of ${files.size} files to validate.")
                     for (file in files) {
                         log.info("Validate file $file.")
                         val format = validator.validateExtension(file)
@@ -78,20 +99,24 @@ class Service(fileName: String = "app.yml") {
                     if (failures > 0) {
                         log.warn("Validation ended with $failures failures!")
                         producer.sendJobReport(
-                            Report(
+                            ProcessReport(
                                 "text-file-validation",
                                 status = ReportStatus.failure,
-                                message = ReportMessages.processFailure(failures, reports.size)
+                                total = totalCount,
+                                failures = failures,
+                                successes = totalCount - failures
                             ),
                             settings.processReportTopic
                         )
                     } else {
                         log.info("Validation was successful!")
                         producer.sendJobReport(
-                            Report(
+                            ProcessReport(
                                 "text-file-validation",
                                 status = ReportStatus.success,
-                                message = ReportMessages.processSuccess(reports.size)
+                                total = totalCount,
+                                successes = reports.size,
+                                failures = 0
                             ),
                             settings.processReportTopic
                         )
@@ -99,20 +124,24 @@ class Service(fileName: String = "app.yml") {
                 } catch (ex: SftpClientException) {
                     ex.printStackTrace()
                     log.error("SFTP Exception: ${ex.localizedMessage}.")
-                    val report = Report(
+                    val report = ProcessReport(
                         "text-file-validation",
                         status = ReportStatus.failure,
-                        message = "SFTP Exception: ${ex.localizedMessage}."
+                        total = totalCount,
+                        failures = reports.count { report -> report.status == ReportStatus.failure },
+                        successes = reports.count { report -> report.status == ReportStatus.success }
                     )
                     producer.sendJobReport(report, settings.processReportTopic)
                 } catch (ex: Exception) {
                     ex.printStackTrace()
                     log.error(ex.javaClass.canonicalName + ": " + ex.localizedMessage)
                     producer.sendJobReport(
-                        Report(
+                        ProcessReport(
                             "text-file-validation",
                             status = ReportStatus.failure,
-                            message = "Unknown Exception: ${ex.javaClass.canonicalName}: ${ex.localizedMessage}."
+                            total = totalCount,
+                            failures = reports.count { report -> report.status == ReportStatus.failure },
+                            successes = reports.count { report -> report.status == ReportStatus.success }
                         ),
                         settings.processReportTopic
                     )

src/test/kotlin/TestParams.kt

@@ -22,6 +22,6 @@ data class TestParams(
     val configFile: String,
     val expectedKey: String,
     val expectedValue: String,
-    val expectedReportValue: String,
-    val expectedProcessReport: Report
+    val expectedReportValue: Report,
+    val expectedProcessReport: ProcessReport
 )