Commit 865239c3 authored by Jonas Waeber's avatar Jonas Waeber
Browse files

Fixed xlsx bug and refactored constant handling

All constant values can now be found inside of a single file.
parent fc3929f8
Pipeline #9241 passed with stages
in 7 minutes and 15 seconds
......@@ -23,7 +23,7 @@ import org.apache.logging.log4j.LogManager
class App {
companion object {
private val log = LogManager.getLogger("App")
private val log = LogManager.getLogger("TextFileValidationApp")
@JvmStatic fun main(args: Array<String>) {
try {
val service = Service()
......
......@@ -26,31 +26,31 @@ import org.memobase.sftp.SftpClient
class FileValidation(private val sftp: SftpClient) {
private val supportedExtensions = mapOf(
Pair("csv", "CSV"),
Pair("tsv", "TSV"),
Pair("xslx", "XSLX"),
Pair("xsl", "XSL")
Pair(Extensions.csv, Formats.csv),
Pair(Extensions.tsv, Formats.tsv),
Pair(Extensions.xlsx, Formats.xlsx),
Pair(Extensions.xls, Formats.xls)
)
fun validate(file: File): Pair<Message, Report> {
return when (val format = validateExtension(file)) {
"CSV", "TSV" -> {
Formats.csv, Formats.tsv -> {
sftp.open(file).use {
val stream = it.RemoteFileInputStream()
try {
csvReader {
charset = "UTF-8"
delimiter = if (format == "CSV") ',' else '\t'
delimiter = if (format == Formats.csv) ',' else '\t'
quoteChar = '"'
escapeChar = '\\'
}.readAll(stream)
} catch (ex: MalformedCSVException) {
return@use Pair(
Message("ERROR", file.path),
Message(Formats.error, file.path),
Report(
id = file.name,
status = "FAILURE",
message = "$format ERROR: " + ex.localizedMessage
status = ReportStatus.failure,
message = ReportMessages.formatError(file.path, format, ex.localizedMessage)
)
)
}
......@@ -58,24 +58,24 @@ class FileValidation(private val sftp: SftpClient) {
Message(format, file.path),
Report(
id = file.name,
status = "SUCCESS",
message = "Validated file at path ${file.path} with format $format."
status = ReportStatus.success,
message = ReportMessages.validatedFile(file.path, format)
)
)
}
}
"XSLX", "XSL" -> {
Formats.xlsx, Formats.xls -> {
sftp.open(file).use {
try {
val stream = it.RemoteFileInputStream()
WorkbookFactory.create(stream).close()
} catch (ex: Exception) {
return@use Pair(
Message("ERROR", file.path),
Message(Formats.error, file.path),
Report(
id = file.name,
status = "FAILURE",
message = "$format ERROR: ${ex.localizedMessage}"
status = ReportStatus.failure,
message = ReportMessages.formatError(file.path, format, ex.localizedMessage)
)
)
}
......@@ -83,24 +83,24 @@ class FileValidation(private val sftp: SftpClient) {
Message(format, file.path),
Report(
id = file.name,
status = "SUCCESS",
message = "Validated file at path ${file.path} with format $format."
status = ReportStatus.success,
message = ReportMessages.validatedFile(file.path, format)
)
)
}
}
else -> Pair(
Message("ERROR", file.path),
Message(Formats.error, file.path),
Report(
id = file.name,
status = "FAILURE",
message = "File Extension Error: Not a valid file extension: ${file.name}."
status = ReportStatus.failure,
message = ReportMessages.invalidFileExtension(file.name)
)
)
}
}
private fun validateExtension(file: File): String {
return supportedExtensions.getOrDefault(file.extension, "INVALID")
return supportedExtensions.getOrDefault(file.extension, Formats.invalid)
}
}
......@@ -53,17 +53,17 @@ class Service(fileName: String = "app.yml") {
producer.sendReport(validationResult.second)
reports.add(validationResult.second)
}
val failures = reports.count { report -> report.status == "FAILURE" }
val failures = reports.count { report -> report.status == ReportStatus.failure }
if (failures > 0) {
log.info("Validation ended with $failures failures!")
log.warn("Validation ended with $failures failures!")
producer.sendJobReport(
Report(settings.id, status = "FAILURE", message = "Failed to validate $failures of ${reports.size} files."),
Report(settings.id, status = ReportStatus.failure, message = ReportMessages.processFailure(failures, reports.size)),
settings.processReportTopic
)
} else {
log.info("Validation was successful!")
producer.sendJobReport(
Report(settings.id, status = "SUCCESS", message = "Successfully validated ${reports.size} files."),
Report(settings.id, status = ReportStatus.success, message = ReportMessages.processSuccess(reports.size)),
settings.processReportTopic
)
}
......
/*
* text-file-validation
* Copyright (C) 2020 Memoriav
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.memobase
object Formats {
const val csv = "CSV"
const val tsv = "TSV"
const val xlsx = "XLSX"
const val xls = "XLS"
const val ods = "ODS"
const val invalid = "INVALID"
const val error = "ERROR"
}
object Extensions {
const val csv = "csv"
const val tsv = "tsv"
const val xlsx = "xlsx"
const val xls = "xls"
const val ods = "ods"
}
object ReportStatus {
const val success = "SUCCESS"
const val failure = "FAILURE"
}
object ReportMessages {
fun processFailure(failures: Int, total: Int): String {
return "Failed to validate $failures of $total files."
}
fun processSuccess(total: Int): String {
return "Successfully validated $total files."
}
fun invalidFileExtension(fileName: String): String {
return "File Extension Error: Not a valid file extension: $fileName."
}
fun validatedFile(path: String, format: String): String {
return "Validated file at path $path with format $format."
}
fun formatError(path: String, format: String, message: String): String {
return "$format ERROR: $message for file $path."
}
}
......@@ -69,7 +69,7 @@ class Tests {
FileInputStream("src/test/resources/data/file.txt")
)
sftpServer.putFile(
"/memobase/test_institution_5/test_record_set_5/file.xslx",
"/memobase/test_institution_5/test_record_set_5/file.xlsx",
FileInputStream("src/test/resources/data/20190906_Brandt_Metadaten.xlsx")
)
}
......@@ -168,7 +168,7 @@ class Tests {
Report(
id = "invalid.csv",
status = "FAILURE",
message = "CSV ERROR: Fields num seems to be 5 on each row, but on 2th csv row, fields num is 7."
message = "CSV ERROR: Fields num seems to be 5 on each row, but on 2th csv row, fields num is 7. for file /memobase/test_institution_3/test_record_set_3/invalid.csv."
)
),
expectedProcessReport = Report(
......@@ -196,13 +196,13 @@ class Tests {
),
TestParams(
"test5.yml",
expectedKey = "file.xslx",
expectedValue = "{\"format\" : \"XSLX\", \"path\" : \"/memobase/test_institution_5/test_record_set_5/file.xslx\"}",
expectedKey = "file.xlsx",
expectedValue = "{\"format\" : \"XLSX\", \"path\" : \"/memobase/test_institution_5/test_record_set_5/file.xlsx\"}",
expectedReportValue = Klaxon().toJsonString(
Report(
id = "file.xslx",
id = "file.xlsx",
status = "SUCCESS",
message = "Validated file at path /memobase/test_institution_5/test_record_set_5/file.xslx with format XSLX."
message = "Validated file at path /memobase/test_institution_5/test_record_set_5/file.xlsx with format XLSX."
)
),
expectedProcessReport = Report(
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment