Commit ac994493 authored by Jonas Waeber's avatar Jonas Waeber
Browse files

Upgrade dependencies

Refactor message & formats to use library versions.
Improve error handling
parent 21daa9ee
Pipeline #33856 passed with stages
in 3 minutes and 19 seconds
plugins {
id 'application'
id 'distribution'
id 'org.jetbrains.kotlin.jvm' version '1.3.71'
id 'com.palantir.git-version' version '0.11.0'
id 'org.jetbrains.kotlin.jvm' version '1.5.30'
id "io.freefair.git-version" version "6.2.0"
id 'org.jlleitschuh.gradle.ktlint' version '9.2.1'
}
group 'org.memobase'
version = gitVersion()
mainClassName = 'org.memobase.App'
jar {
......@@ -27,7 +26,7 @@ repositories {
ext {
kafkaV = '2.7.0'
log4jV = '2.11.2'
log4jV = '2.14.1'
}
dependencies {
......@@ -41,26 +40,26 @@ dependencies {
//implementation "org.apache.kafka:kafka-streams:${kafkaV}"
compile group: 'org.apache.kafka', name: 'kafka-clients', version: kafkaV
implementation 'org.memobase:memobase-service-utilities:3.0.1'
implementation 'org.memobase:memobase-service-utilities:3.1.0'
// SFTP Client
// is needed because of a bug.
implementation 'com.hierynomus:sshj:0.27.0'
// JSON Parser
implementation 'com.beust:klaxon:5.5'
// CSV Reader
implementation("com.github.doyaaaaaken:kotlin-csv-jvm:0.7.3")
implementation('com.github.doyaaaaaken:kotlin-csv-jvm:1.1.0')
// XSLX / XSL Reader
implementation 'org.apache.poi:poi:4.1.2'
implementation 'org.apache.poi:poi-ooxml:4.1.2'
implementation 'org.apache.poi:poi:5.0.0'
implementation 'org.apache.poi:poi-ooxml:5.0.0'
// KOTLIN IMPORTS
implementation 'org.jetbrains.kotlin:kotlin-stdlib-jdk8'
implementation "org.jetbrains.kotlin:kotlin-script-runtime:1.3.71"
implementation "org.jetbrains.kotlin:kotlin-reflect:1.3.71"
implementation "org.jetbrains.kotlin:kotlin-script-runtime:1.5.30"
implementation "org.jetbrains.kotlin:kotlin-reflect:1.5.30"
// JUNIT
testCompile("org.junit.jupiter:junit-jupiter:5.4.2")
testImplementation 'org.assertj:assertj-core:3.15.0'
testCompile('org.junit.jupiter:junit-jupiter:5.8.1')
testImplementation 'org.assertj:assertj-core:3.21.0'
}
compileKotlin {
......
......@@ -18,21 +18,18 @@
package org.memobase
import kotlin.system.exitProcess
import org.apache.logging.log4j.LogManager
class App {
companion object {
private val log = LogManager.getLogger(this::class.java)
@JvmStatic fun main(args: Array<String>) {
@JvmStatic
fun main(args: Array<String>) {
try {
val service = Service()
service.run()
exitProcess(0)
} catch (ex: Exception) {
ex.printStackTrace()
log.error("Unexpected Exception Thrown (shutting down service): " + ex.message)
exitProcess(1)
}
}
}
......
......@@ -20,6 +20,8 @@ package org.memobase
import ch.memobase.reporting.Report
import ch.memobase.reporting.ReportStatus
import ch.memobase.utility.AcceptedFileFormat
import ch.memobase.utility.TextFileValidationMessage
import com.beust.klaxon.Klaxon
import com.beust.klaxon.KlaxonException
import com.github.doyaaaaaken.kotlincsv.dsl.csvReader
......@@ -37,33 +39,33 @@ class FileValidation(private val step: String) {
private val klaxon = Klaxon()
private val supportedExtensions = mapOf(
Pair(Extensions.csv, Formats.csv),
Pair(Extensions.tsv, Formats.tsv),
Pair(Extensions.xlsx, Formats.xlsx),
Pair(Extensions.xls, Formats.xls),
Pair(Extensions.xml, Formats.xml),
Pair(Extensions.json, Formats.json)
Pair(Extensions.csv, AcceptedFileFormat.CSV),
Pair(Extensions.tsv, AcceptedFileFormat.TSV),
Pair(Extensions.xlsx, AcceptedFileFormat.XLSX),
Pair(Extensions.xls, AcceptedFileFormat.XLS),
Pair(Extensions.xml, AcceptedFileFormat.XML),
Pair(Extensions.json, AcceptedFileFormat.JSON)
)
fun validateExtension(file: File): String {
return supportedExtensions.getOrDefault(file.extension, Formats.invalid)
fun validateExtension(file: File): AcceptedFileFormat {
return supportedExtensions.getOrDefault(file.extension, AcceptedFileFormat.INVALID)
}
fun validate(inputStream: InputStream, format: String, file: File): Pair<Message, Report> {
fun validate(inputStream: InputStream, format: AcceptedFileFormat, file: File): Pair<TextFileValidationMessage, Report> {
log.info("Validate file with format $format.")
return when (format) {
Formats.csv, Formats.tsv -> {
AcceptedFileFormat.CSV, AcceptedFileFormat.TSV -> {
inputStream.use { stream ->
try {
csvReader {
charset = "UTF-8"
delimiter = if (format == Formats.csv) ',' else '\t'
delimiter = if (format == AcceptedFileFormat.CSV) ',' else '\t'
quoteChar = '"'
escapeChar = '\\'
}.readAll(stream)
} catch (ex: MalformedCSVException) {
return@use Pair(
Message(Formats.error, file.path),
TextFileValidationMessage(AcceptedFileFormat.ERROR, file.path),
Report(
id = file.name,
status = ReportStatus.fatal,
......@@ -73,7 +75,7 @@ class FileValidation(private val step: String) {
)
}
Pair(
Message(format, file.path),
TextFileValidationMessage(format, file.path),
Report(
id = file.name,
status = ReportStatus.success,
......@@ -83,13 +85,13 @@ class FileValidation(private val step: String) {
)
}
}
Formats.xlsx, Formats.xls -> {
AcceptedFileFormat.XLSX, AcceptedFileFormat.XLS -> {
inputStream.use { stream ->
try {
WorkbookFactory.create(stream).close()
} catch (ex: Exception) {
return Pair(
Message(Formats.error, file.path),
TextFileValidationMessage(AcceptedFileFormat.ERROR, file.path),
Report(
id = file.name,
status = ReportStatus.fatal,
......@@ -99,7 +101,7 @@ class FileValidation(private val step: String) {
)
}
Pair(
Message(format, file.path),
TextFileValidationMessage(format, file.path),
Report(
id = file.name,
status = ReportStatus.success,
......@@ -109,14 +111,14 @@ class FileValidation(private val step: String) {
)
}
}
Formats.xml -> {
AcceptedFileFormat.XML -> {
inputStream.use {
val dbFactory = DocumentBuilderFactory.newInstance()
val dBuilder = dbFactory.newDocumentBuilder()
try {
dBuilder.parse(it)
Pair(
Message(format, file.path), Report(
TextFileValidationMessage(format, file.path), Report(
id = file.name,
status = ReportStatus.success,
message = ReportMessages.validatedFile(file.path, format),
......@@ -125,7 +127,7 @@ class FileValidation(private val step: String) {
)
} catch (ex: SAXException) {
Pair(
Message(Formats.error, file.path),
TextFileValidationMessage(AcceptedFileFormat.ERROR, file.path),
Report(
id = file.name,
status = ReportStatus.fatal,
......@@ -135,7 +137,7 @@ class FileValidation(private val step: String) {
)
} catch (ex: IOException) {
Pair(
Message(Formats.error, file.path),
TextFileValidationMessage(AcceptedFileFormat.ERROR, file.path),
Report(
id = file.name,
status = ReportStatus.fatal,
......@@ -146,13 +148,13 @@ class FileValidation(private val step: String) {
}
}
}
Formats.json -> {
AcceptedFileFormat.JSON -> {
inputStream.use { iS ->
try {
val output = klaxon.parseJsonObject(iS.bufferedReader(Charsets.UTF_8))
if (output.isEmpty()) {
Pair(
Message(format, file.path), Report(
TextFileValidationMessage(format, file.path), Report(
id = file.name,
status = ReportStatus.fatal,
message = ReportMessages.formatError(file.path, format, "Parsed object is empty"),
......@@ -161,7 +163,7 @@ class FileValidation(private val step: String) {
)
} else {
Pair(
Message(format, file.path), Report(
TextFileValidationMessage(format, file.path), Report(
id = file.name,
status = ReportStatus.success,
message = ReportMessages.validatedFile(file.path, format),
......@@ -171,7 +173,7 @@ class FileValidation(private val step: String) {
}
} catch (ex: KlaxonException) {
Pair(
Message(Formats.error, file.path),
TextFileValidationMessage(AcceptedFileFormat.ERROR, file.path),
Report(
id = file.name,
status = ReportStatus.fatal,
......@@ -181,7 +183,7 @@ class FileValidation(private val step: String) {
)
} catch (ex: ClassCastException) {
Pair(
Message(Formats.error, file.path),
TextFileValidationMessage(AcceptedFileFormat.ERROR, file.path),
Report(
id = file.name,
status = ReportStatus.fatal,
......@@ -193,7 +195,7 @@ class FileValidation(private val step: String) {
}
}
else -> Pair(
Message(Formats.error, file.path),
TextFileValidationMessage(AcceptedFileFormat.ERROR, file.path),
Report(
id = file.name,
status = ReportStatus.fatal,
......
/*
* text-file-validation
* Copyright (C) 2020-2021 Memoriav
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.memobase
object Formats {
const val csv = "CSV"
const val tsv = "TSV"
const val xlsx = "XLSX"
const val xls = "XLS"
const val xml = "XML"
const val json = "JSON"
const val invalid = "INVALID"
const val error = "ERROR"
}
\ No newline at end of file
/*
* text-file-validation
* Copyright (C) 2020-2021 Memoriav
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.memobase
import com.beust.klaxon.Klaxon
data class Message(
val format: String,
val path: String
) {
fun toJson(): String {
return Klaxon().toJsonString(this)
}
}
......@@ -24,6 +24,7 @@ import org.apache.kafka.clients.producer.ProducerRecord
import org.apache.kafka.common.header.Header
import org.apache.kafka.common.header.internals.RecordHeader
import ch.memobase.reporting.Report
import ch.memobase.utility.TextFileValidationMessage
class Producer(
props: Properties,
......@@ -39,7 +40,7 @@ class Producer(
RecordHeader(item.key as String, (item.value as String).toByteArray())
}
}
fun sendMessage(key: String, message: Message) {
fun sendMessage(key: String, message: TextFileValidationMessage) {
instance.send(ProducerRecord(outputTopic, null, key, message.toJson(), headers))
}
......
......@@ -18,9 +18,11 @@
package org.memobase
import ch.memobase.utility.AcceptedFileFormat
object ReportMessages {
// SUCCESS
fun validatedFile(path: String, format: String): String {
fun validatedFile(path: String, format: AcceptedFileFormat): String {
return "Validated file at path $path with format $format."
}
......@@ -33,7 +35,7 @@ object ReportMessages {
return "SFTP Exception: $message."
}
fun formatError(path: String, format: String, message: String): String {
fun formatError(path: String, format: AcceptedFileFormat, message: String): String {
return "$format ERROR: $message for file $path."
}
......
......@@ -104,7 +104,6 @@ class Service(fileName: String = "app.yml") {
totalCount = fileList.size
fileList
} catch (ex: SftpClientException) {
ex.printStackTrace()
log.error("SFTP Exception: Could not compile the file list on sftp server in directory: '$basePath'.")
val report = Report(
"$recordSetId#$sessionId",
......@@ -113,7 +112,7 @@ class Service(fileName: String = "app.yml") {
step = step
)
producer.sendReport(report)
exitProcess(1)
return
}
log.info("Retrieved file list from sftp server from folder: $basePath")
var reportCount = 0
......@@ -147,7 +146,6 @@ class Service(fileName: String = "app.yml") {
}
log.info("Validated a total of $reportCount files with $sftpClientExceptions SFTP Client Exceptions")
} catch (ex: SftpClientException) {
ex.printStackTrace()
log.error("SFTP Exception: ${ex.localizedMessage}.")
producer.sendReport(
Report(
......@@ -159,7 +157,6 @@ class Service(fileName: String = "app.yml") {
)
exitProcess(1)
} catch (ex: Exception) {
ex.printStackTrace()
log.error("${ex.javaClass.canonicalName}: ${ex.localizedMessage}.")
producer.sendReport(
Report(
......
......@@ -19,13 +19,13 @@ package org.memobase.test
import ch.memobase.reporting.Report
import ch.memobase.reporting.ReportStatus
import ch.memobase.utility.AcceptedFileFormat
import ch.memobase.utility.TextFileValidationMessage
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.TestInstance
import org.junit.jupiter.api.assertAll
import org.memobase.FileValidation
import org.memobase.Formats
import org.memobase.Message
import java.io.File
import java.io.FileInputStream
import java.io.InputStream
......@@ -35,13 +35,13 @@ class TestFileValidation {
private val path = "src/test/resources/data"
private fun read(format: String, filename: String): InputStream {
private fun read(format: AcceptedFileFormat, filename: String): InputStream {
return FileInputStream(File("$path/$format/$filename"))
}
private fun assertValidation(
result: Pair<Message, Report>,
expectedFormat: String,
result: Pair<TextFileValidationMessage, Report>,
expectedFormat: AcceptedFileFormat,
expectedPath: String,
expectedStatus: String,
expectedMessage: String
......@@ -71,13 +71,13 @@ class TestFileValidation {
fun `test valid json file`() {
val validation = FileValidation("test")
val result = validation.validate(
read("json", "valid.json"),
Formats.json,
read(AcceptedFileFormat.JSON, "valid.json"),
AcceptedFileFormat.JSON,
File("$path/json/valid.json")
)
assertValidation(
result,
Formats.json,
AcceptedFileFormat.JSON,
"src/test/resources/data/json/valid.json",
ReportStatus.success,
"Validated file at path src/test/resources/data/json/valid.json with format JSON."
......@@ -88,13 +88,13 @@ class TestFileValidation {
fun `test empty json file`() {
val validation = FileValidation("test")
val result = validation.validate(
read("json", "empty.json"),
Formats.json,
read(AcceptedFileFormat.JSON, "empty.json"),
AcceptedFileFormat.JSON,
File("$path/json/empty.json")
)
assertValidation(
result,
Formats.json,
AcceptedFileFormat.JSON,
"src/test/resources/data/json/empty.json",
ReportStatus.fatal,
"JSON ERROR: Parsed object is empty for file src/test/resources/data/json/empty.json."
......@@ -105,13 +105,13 @@ class TestFileValidation {
fun `test invalid json file`() {
val validation = FileValidation("test")
val result = validation.validate(
read("json", "invalid.json"),
Formats.json,
read(AcceptedFileFormat.JSON, "invalid.json"),
AcceptedFileFormat.JSON,
File("$path/json/invalid.json")
)
assertValidation(
result,
Formats.error,
AcceptedFileFormat.ERROR,
"src/test/resources/data/json/invalid.json",
ReportStatus.fatal,
"JSON ERROR: Unexpected character at position 5: 'a' (ASCII: 97)' for file src/test/resources/data/json/invalid.json."
......@@ -122,13 +122,13 @@ class TestFileValidation {
fun `test list json file`() {
val validation = FileValidation("test")
val result = validation.validate(
read("json", "list.json"),
Formats.json,
read(AcceptedFileFormat.JSON, "list.json"),
AcceptedFileFormat.JSON,
File("$path/json/list.json")
)
assertValidation(
result,
Formats.error,
AcceptedFileFormat.ERROR,
"src/test/resources/data/json/list.json",
ReportStatus.fatal,
"JSON ERROR: Expected json object as top level entity for file src/test/resources/data/json/list.json."
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment