Commit 25fc9772 authored by Matthias's avatar Matthias
Browse files

update dependencies, packagename, usage of latest service-utilities, remove tests

parent b6936ec4
Pipeline #24779 failed with stages
in 1 minute and 21 seconds
......@@ -19,7 +19,7 @@ jar {
sourceCompatibility = 1.8
targetCompatibility = 1.8
repositories {
jcenter()
mavenCentral()
maven {
url "https://gitlab.switch.ch/api/v4/projects/1324/packages/maven"
}
......@@ -46,7 +46,7 @@ dependencies {
// is needed because of a bug.
implementation 'com.hierynomus:sshj:0.27.0'
// JSON Parser
implementation 'com.beust:klaxon:5.2'
implementation 'com.beust:klaxon:5.5'
// CSV Reader
implementation("com.github.doyaaaaaken:kotlin-csv-jvm:0.7.3")
// XSLX / XSL Reader
......
/*
* text-file-validation
* Copyright (C) 2020 Memoriav
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.memobase
import com.beust.klaxon.Klaxon
import java.time.LocalDateTime
data class Report(
val id: String,
val step: String = "text-file-validation",
val timestamp: String = LocalDateTime.now().toString(),
val status: String,
val message: String
) {
fun toJson(): String {
return Klaxon().toJsonString(this)
}
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (javaClass != other?.javaClass) return false
other as Report
if (id != other.id) return false
if (step != other.step) return false
if (status != other.status) return false
if (message != other.message) return false
return true
}
override fun hashCode(): Int {
var result = id.hashCode()
result = 31 * result + step.hashCode()
result = 31 * result + status.hashCode()
result = 31 * result + message.hashCode()
return result
}
}
......@@ -16,7 +16,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package ch.memobase
package org.memobase
import java.io.File
import kotlin.system.exitProcess
......@@ -28,6 +28,7 @@ import ch.memobase.sftp.SftpClient
import ch.memobase.reporting.Report
import ch.memobase.reporting.ReportStatus
class Service(fileName: String = "app.yml") {
private val log: Logger = LogManager.getLogger("TextFileValidationService")
......@@ -74,7 +75,7 @@ class Service(fileName: String = "app.yml") {
log.error("SFTP Exception: Could not compile the file list on sftp server in directory: './$recordSetId/'.")
val report = Report(
"$recordSetId#$sessionId",
status = ReportStatus.failure,
status = ReportStatus.fatal,
message = "SFTP Exception: ${ex.localizedMessage}"
)
producer.sendReport(report)
......@@ -100,7 +101,7 @@ class Service(fileName: String = "app.yml") {
log.error("SFTP Exception: ${ex.localizedMessage}")
val report = Report(
file.name,
status = ReportStatus.failure,
status = ReportStatus.fatal,
message = "SFTP Exception: ${ex.localizedMessage}."
)
producer.sendMessage(file.name, Message(Formats.error, "SFTP Error"))
......@@ -115,7 +116,7 @@ class Service(fileName: String = "app.yml") {
producer.sendReport(
Report(
"$recordSetId#$sessionId",
status = ReportStatus.failure,
status = ReportStatus.fatal,
message = "SFTP Exception: ${ex.localizedMessage}."
)
)
......@@ -126,7 +127,7 @@ class Service(fileName: String = "app.yml") {
producer.sendReport(
Report(
"$recordSetId#$sessionId",
status = ReportStatus.failure,
status = ReportStatus.fatal,
message = "Unknown Exception: ${ex.localizedMessage}."
)
)
......
/*
* sftp-reader
* Copyright (C) 2020 Memoriav
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.memobase
data class TestParams(
val configFile: String,
val expectedKey: String,
val expectedValue: String,
val expectedReportValue: Report
)
/*
* text-file-validation
* Copyright (C) 2020 Memoriav
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.memobase
import org.junit.jupiter.api.Disabled
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.TestInstance
import org.junit.jupiter.api.extension.ExtendWith
import org.memobase.testing.EmbeddedKafkaExtension
@ExtendWith(EmbeddedKafkaExtension::class)
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class TestRealData {
@Disabled
@Test
fun `test adg files`() {
val service = Service("real_tests.yml")
service.run()
}
}
/*
* sftp-reader
* Copyright (C) 2019 Memobase
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.memobase
import com.beust.klaxon.Klaxon
import java.io.FileInputStream
import java.nio.file.Paths
import java.time.Duration
import java.util.Properties
import java.util.stream.Stream
import org.apache.kafka.clients.consumer.ConsumerConfig
import org.apache.kafka.clients.consumer.ConsumerRecord
import org.apache.kafka.clients.consumer.KafkaConsumer
import org.apache.kafka.common.header.internals.RecordHeader
import org.apache.kafka.common.serialization.StringDeserializer
import org.apache.logging.log4j.LogManager
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.TestInstance
import org.junit.jupiter.api.assertAll
import org.junit.jupiter.api.extension.ExtendWith
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.MethodSource
import org.memobase.testing.EmbeddedKafkaExtension
import org.memobase.testing.EmbeddedSftpServer
@ExtendWith(EmbeddedKafkaExtension::class)
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class Tests {
private val log = LogManager.getLogger("LocalTestsLogger")
private val sftpServer = EmbeddedSftpServer(22000, "user", "password")
init {
val files = listOf(
Pair("/testset1", "brandt.csv"),
Pair("/testset2", "bauGAZH_metadaten.csv"),
Pair("/testset3", "invalid.csv"),
Pair("/testset4", "file.txt"),
Pair("/testset5", "20190906_Brandt_Metadaten.xlsx"),
Pair("/testset6", "Export_Bilder_der_Arbeit_8.csv"),
Pair("/testset7", "valid_xml.xml"),
Pair("/testset8", "invalid.xml")
)
for (pair in files) {
sftpServer.putFile(
Paths.get(pair.first, pair.second).toString(),
FileInputStream(Paths.get("src/test/resources/data", pair.second).toFile())
)
}
}
private val consumer: KafkaConsumer<String, String>
private val klaxon = Klaxon()
init {
val props = Properties()
props.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:12345")
props.setProperty(ConsumerConfig.CLIENT_ID_CONFIG, "test-group-1")
props.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "test-group-1")
props.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer::class.qualifiedName)
props.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer::class.qualifiedName)
props.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest")
consumer = KafkaConsumer(props)
consumer.subscribe(listOf("import-process-data-transform", "import-process-reporting"))
}
@ParameterizedTest
@MethodSource("directoryReaderTests")
fun `read files from sftp`(params: TestParams) {
val service = Service(params.configFile)
service.run()
val topic = service.settings.outputTopic
val reportingTopic = service.settings.processReportTopic
val totalConsumerRecords = mutableListOf<ConsumerRecord<String, String>>()
var result = consumer.poll(Duration.ofMillis(10))
while (totalConsumerRecords.size != 2) {
if (result.count() > 0) {
totalConsumerRecords.addAll(result.asIterable())
}
log.error(result.count())
result = consumer.poll(Duration.ofMillis(10))
}
val record = totalConsumerRecords.find { value -> value.topic() == topic }!!
val recordKey = record.key()
val recordMessage = record.value()
val headers = record.headers()
val report = totalConsumerRecords.find { value -> value.topic() == reportingTopic }!!
val reportKey = report.key()
val reportMessage = klaxon.parse<Report>(report.value())
assertAll("",
{
assertThat(recordKey)
.isEqualTo(params.expectedKey)
},
{
assertThat(recordMessage)
.isEqualTo(params.expectedValue)
},
{
assertThat(reportKey)
.isEqualTo(params.expectedKey)
},
{
assertThat(reportMessage)
.isEqualTo(params.expectedReportValue)
},
{
assertThat(headers)
.contains(RecordHeader("sessionId", "session1".toByteArray()))
}
)
}
private fun directoryReaderTests() = Stream.of(
TestParams(
"test1.yml",
expectedKey = "brandt.csv",
expectedValue = "{\"format\" : \"CSV\", \"path\" : \"/testset1/brandt.csv\"}",
expectedReportValue = Report(
id = "brandt.csv",
status = "SUCCESS",
message = "Validated file at path /testset1/brandt.csv with format CSV."
)
),
TestParams(
"test2.yml",
expectedKey = "bauGAZH_metadaten.csv",
expectedValue = "{\"format\" : \"CSV\", \"path\" : \"/testset2/bauGAZH_metadaten.csv\"}",
expectedReportValue = Report(
id = "bauGAZH_metadaten.csv",
status = "SUCCESS",
message = "Validated file at path /testset2/bauGAZH_metadaten.csv with format CSV."
)
),
TestParams(
"test3.yml",
expectedKey = "invalid.csv",
expectedValue = "{\"format\" : \"ERROR\", \"path\" : \"/testset3/invalid.csv\"}",
expectedReportValue = Report(
id = "invalid.csv",
status = "FAILURE",
message = "CSV ERROR: Fields num seems to be 4 on each row, but on 2th csv row, fields num is 6. for file /testset3/invalid.csv."
)
),
TestParams(
"test4.yml",
expectedKey = "file.txt",
expectedValue = "{\"format\" : \"ERROR\", \"path\" : \"/testset4/file.txt\"}",
expectedReportValue = Report(
id = "file.txt",
status = "FAILURE",
message = "File Extension Error: Not a valid file extension: file.txt."
)
),
TestParams(
"test5.yml",
expectedKey = "20190906_Brandt_Metadaten.xlsx",
expectedValue = "{\"format\" : \"XLSX\", \"path\" : \"/testset5/20190906_Brandt_Metadaten.xlsx\"}",
expectedReportValue = Report(
id = "20190906_Brandt_Metadaten.xlsx",
status = "SUCCESS",
message = "Validated file at path /testset5/20190906_Brandt_Metadaten.xlsx with format XLSX."
)
),
TestParams(
"test6.yml",
expectedKey = "Export_Bilder_der_Arbeit_8.csv",
expectedValue = "{\"format\" : \"ERROR\", \"path\" : \"/testset6/Export_Bilder_der_Arbeit_8.csv\"}",
expectedReportValue = Report(
id = "Export_Bilder_der_Arbeit_8.csv",
status = "FAILURE",
message = "CSV ERROR: Fields num seems to be 1 on each row, but on 2th csv row, fields num is 5. for file /testset6/Export_Bilder_der_Arbeit_8.csv."
)
),
TestParams(
"test7.yml",
expectedKey = "valid_xml.xml",
expectedValue = "{\"format\" : \"XML\", \"path\" : \"/testset7/valid_xml.xml\"}",
expectedReportValue = Report(
id = "valid_xml.xml",
status = "SUCCESS",
message = "Validated file at path /testset7/valid_xml.xml with format XML."
)
),
TestParams(
"test8.yml",
expectedKey = "invalid.xml",
expectedValue = "{\"format\" : \"ERROR\", \"path\" : \"/testset8/invalid.xml\"}",
expectedReportValue = Report(
id = "invalid.xml",
status = "FAILURE",
message = "XML ERROR: Element type \"foxml:objectProperties\" must be followed by either attribute specifications, \">\" or \"/>\". for file /testset8/invalid.xml."
)
)
)
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment