Tests.kt 10.9 KB
Newer Older
Jonas Waeber's avatar
Jonas Waeber committed
1
/*
Jonas Waeber's avatar
Jonas Waeber committed
2
 * sftp-reader
Jonas Waeber's avatar
Jonas Waeber committed
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
 * Copyright (C) 2019  Memobase
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
18
package org.memobase
Jonas Waeber's avatar
Jonas Waeber committed
19

Jonas Waeber's avatar
Jonas Waeber committed
20
import java.io.FileInputStream
Jonas Waeber's avatar
Jonas Waeber committed
21
import java.nio.file.Paths
Jonas Waeber's avatar
Jonas Waeber committed
22 23 24 25
import java.time.Duration
import java.util.Properties
import java.util.stream.Stream
import org.apache.kafka.clients.consumer.ConsumerConfig
Jonas Waeber's avatar
Jonas Waeber committed
26
import org.apache.kafka.clients.consumer.ConsumerRecord
Jonas Waeber's avatar
Jonas Waeber committed
27 28
import org.apache.kafka.clients.consumer.KafkaConsumer
import org.apache.kafka.common.serialization.StringDeserializer
Jonas Waeber's avatar
Jonas Waeber committed
29
import org.apache.logging.log4j.LogManager
Jonas Waeber's avatar
Jonas Waeber committed
30
import org.assertj.core.api.Assertions.assertThat
Jonas Waeber's avatar
Jonas Waeber committed
31
import org.junit.jupiter.api.TestInstance
Jonas Waeber's avatar
Jonas Waeber committed
32
import org.junit.jupiter.api.assertAll
33
import org.junit.jupiter.api.extension.ExtendWith
Jonas Waeber's avatar
Jonas Waeber committed
34 35
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.MethodSource
Jonas Waeber's avatar
Jonas Waeber committed
36 37
import org.memobase.testing.EmbeddedKafkaExtension
import org.memobase.testing.EmbeddedSftpServer
Jonas Waeber's avatar
Jonas Waeber committed
38

Jonas Waeber's avatar
Jonas Waeber committed
39
@ExtendWith(EmbeddedKafkaExtension::class)
Jonas Waeber's avatar
Jonas Waeber committed
40
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
Jonas Waeber's avatar
Jonas Waeber committed
41
class Tests {
Jonas Waeber's avatar
Jonas Waeber committed
42
    private val log = LogManager.getLogger("LocalTestsLogger")
Jonas Waeber's avatar
Jonas Waeber committed
43

Jonas Waeber's avatar
Jonas Waeber committed
44 45 46
    private val sftpServer = EmbeddedSftpServer(22000, "user", "password")

    init {
Jonas Waeber's avatar
Jonas Waeber committed
47 48 49 50 51
        val files = listOf(
            Pair("/memobase/test_record_set_1", "brandt.csv"),
            Pair("/memobase/test_record_set_2", "bauGAZH_metadaten.csv"),
            Pair("/memobase/test_record_set_3", "invalid.csv"),
            Pair("/memobase/test_record_set_4", "file.txt"),
Jonas Waeber's avatar
Jonas Waeber committed
52
            Pair("/memobase/test_record_set_5", "20190906_Brandt_Metadaten.xlsx"),
53 54 55
            Pair("/memobase/test_record_set_6", "Export_Bilder_der_Arbeit_8.csv"),
            Pair("/memobase/test_record_set_7", "valid_xml.xml"),
            Pair("/memobase/test_record_set_8", "invalid.xml")
Jonas Waeber's avatar
Jonas Waeber committed
56
        )
Jonas Waeber's avatar
Jonas Waeber committed
57 58

        for (pair in files) {
Jonas Waeber's avatar
Jonas Waeber committed
59 60 61 62
            sftpServer.putFile(
                Paths.get(pair.first, pair.second).toString(),
                FileInputStream(Paths.get("src/test/resources/data", pair.second).toFile())
            )
Jonas Waeber's avatar
Jonas Waeber committed
63
        }
Jonas Waeber's avatar
Jonas Waeber committed
64 65 66 67 68 69 70 71 72 73 74 75 76
    }

    private val consumer: KafkaConsumer<String, String>

    init {
        val props = Properties()
        props.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:12345")
        props.setProperty(ConsumerConfig.CLIENT_ID_CONFIG, "test-group-1")
        props.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "test-group-1")
        props.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer::class.qualifiedName)
        props.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer::class.qualifiedName)
        props.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest")
        consumer = KafkaConsumer(props)
Jonas Waeber's avatar
Jonas Waeber committed
77
        consumer.subscribe(listOf("sftp-reader-p1-j1", "sftp-reader-p1-j1-reporting", "p1-reporting"))
Jonas Waeber's avatar
Jonas Waeber committed
78 79 80 81
    }

    @ParameterizedTest
    @MethodSource("directoryReaderTests")
Jonas Waeber's avatar
Jonas Waeber committed
82
    fun `read files from sftp`(params: TestParams) {
83 84
        val service = Service(params.configFile)
        service.run()
Jonas Waeber's avatar
Jonas Waeber committed
85

86 87 88
        val topic = service.settings.outputTopic
        val reportingTopic = service.settings.outputTopic + "-reporting"
        val processReportingTopic = service.settings.processReportTopic
Jonas Waeber's avatar
Jonas Waeber committed
89

Jonas Waeber's avatar
Jonas Waeber committed
90 91
        val totalConsumerRecords = mutableListOf<ConsumerRecord<String, String>>()

Jonas Waeber's avatar
Jonas Waeber committed
92
        var result = consumer.poll(Duration.ofMillis(10))
Jonas Waeber's avatar
Jonas Waeber committed
93 94 95 96 97
        while (totalConsumerRecords.size != 3) {
            if (result.count() > 0) {
                totalConsumerRecords.addAll(result.asIterable())
            }
            log.error(result.count())
Jonas Waeber's avatar
Jonas Waeber committed
98 99 100
            result = consumer.poll(Duration.ofMillis(10))
        }

Jonas Waeber's avatar
Jonas Waeber committed
101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120
        assertAll("",
            {
                assertThat(totalConsumerRecords.find { value -> value.topic() == topic })
                    .describedAs("Message Test")
                    .hasFieldOrPropertyWithValue("key", params.expectedKey)
                    .hasFieldOrPropertyWithValue("value", params.expectedValue)
            },
            {
                assertThat(totalConsumerRecords.find { value -> value.topic() == reportingTopic })
                    .describedAs("Report Test")
                    .hasFieldOrPropertyWithValue("key", params.expectedKey)
                    .hasFieldOrPropertyWithValue("value", params.expectedReportValue.toJson())
            },
            {
                assertThat(totalConsumerRecords.find { value -> value.topic() == processReportingTopic })
                    .describedAs("Process Report Test")
                    .hasFieldOrPropertyWithValue("key", params.expectedProcessReport.id)
                    .hasFieldOrPropertyWithValue("value", params.expectedProcessReport.toJson())
            }
        )
Jonas Waeber's avatar
Jonas Waeber committed
121
    }
Jonas Waeber's avatar
Jonas Waeber committed
122 123

    private fun directoryReaderTests() = Stream.of(
124 125 126
        TestParams(
            "test1.yml",
            expectedKey = "brandt.csv",
Jonas Waeber's avatar
Jonas Waeber committed
127
            expectedValue = "{\"format\" : \"CSV\", \"path\" : \"/memobase/test_record_set_1/brandt.csv\"}",
Jonas Waeber's avatar
Jonas Waeber committed
128 129 130 131
            expectedReportValue = Report(
                id = "brandt.csv",
                status = "SUCCESS",
                message = "Validated file at path /memobase/test_record_set_1/brandt.csv with format CSV."
Jonas Waeber's avatar
Jonas Waeber committed
132
            ),
Jonas Waeber's avatar
Jonas Waeber committed
133
            expectedProcessReport = ProcessReport(
Jonas Waeber's avatar
Jonas Waeber committed
134
                id = "text-file-validation",
Jonas Waeber's avatar
Jonas Waeber committed
135 136 137 138
                status = ReportStatus.success,
                total = 1,
                failures = 0,
                successes = 1
139 140 141 142 143
            )
        ),
        TestParams(
            "test2.yml",
            expectedKey = "bauGAZH_metadaten.csv",
Jonas Waeber's avatar
Jonas Waeber committed
144
            expectedValue = "{\"format\" : \"CSV\", \"path\" : \"/memobase/test_record_set_2/bauGAZH_metadaten.csv\"}",
Jonas Waeber's avatar
Jonas Waeber committed
145 146 147 148
            expectedReportValue = Report(
                id = "bauGAZH_metadaten.csv",
                status = "SUCCESS",
                message = "Validated file at path /memobase/test_record_set_2/bauGAZH_metadaten.csv with format CSV."
149
            ),
Jonas Waeber's avatar
Jonas Waeber committed
150
            expectedProcessReport = ProcessReport(
Jonas Waeber's avatar
Jonas Waeber committed
151
                id = "text-file-validation",
152
                status = "SUCCESS",
Jonas Waeber's avatar
Jonas Waeber committed
153 154 155
                total = 1,
                failures = 0,
                successes = 1
156 157 158 159 160
            )
        ),
        TestParams(
            "test3.yml",
            expectedKey = "invalid.csv",
Jonas Waeber's avatar
Jonas Waeber committed
161
            expectedValue = "{\"format\" : \"ERROR\", \"path\" : \"/memobase/test_record_set_3/invalid.csv\"}",
Jonas Waeber's avatar
Jonas Waeber committed
162 163 164 165
            expectedReportValue = Report(
                id = "invalid.csv",
                status = "FAILURE",
                message = "CSV ERROR: Fields num seems to be 4 on each row, but on 2th csv row, fields num is 6. for file /memobase/test_record_set_3/invalid.csv."
166
            ),
Jonas Waeber's avatar
Jonas Waeber committed
167
            expectedProcessReport = ProcessReport(
Jonas Waeber's avatar
Jonas Waeber committed
168
                id = "text-file-validation",
169
                status = "FAILURE",
Jonas Waeber's avatar
Jonas Waeber committed
170 171 172
                total = 1,
                failures = 1,
                successes = 0
173 174 175 176 177
            )
        ),
        TestParams(
            "test4.yml",
            expectedKey = "file.txt",
Jonas Waeber's avatar
Jonas Waeber committed
178
            expectedValue = "{\"format\" : \"ERROR\", \"path\" : \"/memobase/test_record_set_4/file.txt\"}",
Jonas Waeber's avatar
Jonas Waeber committed
179 180 181 182
            expectedReportValue = Report(
                id = "file.txt",
                status = "FAILURE",
                message = "File Extension Error: Not a valid file extension: file.txt."
183
            ),
Jonas Waeber's avatar
Jonas Waeber committed
184
            expectedProcessReport = ProcessReport(
Jonas Waeber's avatar
Jonas Waeber committed
185
                id = "text-file-validation",
186
                status = "FAILURE",
Jonas Waeber's avatar
Jonas Waeber committed
187 188 189
                total = 1,
                failures = 1,
                successes = 0
Jonas Waeber's avatar
Jonas Waeber committed
190
            )
191 192 193
        ),
        TestParams(
            "test5.yml",
Jonas Waeber's avatar
Jonas Waeber committed
194 195
            expectedKey = "20190906_Brandt_Metadaten.xlsx",
            expectedValue = "{\"format\" : \"XLSX\", \"path\" : \"/memobase/test_record_set_5/20190906_Brandt_Metadaten.xlsx\"}",
Jonas Waeber's avatar
Jonas Waeber committed
196 197 198 199
            expectedReportValue = Report(
                id = "20190906_Brandt_Metadaten.xlsx",
                status = "SUCCESS",
                message = "Validated file at path /memobase/test_record_set_5/20190906_Brandt_Metadaten.xlsx with format XLSX."
200
            ),
Jonas Waeber's avatar
Jonas Waeber committed
201
            expectedProcessReport = ProcessReport(
Jonas Waeber's avatar
Jonas Waeber committed
202
                id = "text-file-validation",
203
                status = "SUCCESS",
Jonas Waeber's avatar
Jonas Waeber committed
204 205 206
                total = 1,
                failures = 0,
                successes = 1
207
            )
Jonas Waeber's avatar
Jonas Waeber committed
208 209 210 211 212
        ),
        TestParams(
            "test6.yml",
            expectedKey = "Export_Bilder_der_Arbeit_8.csv",
            expectedValue = "{\"format\" : \"ERROR\", \"path\" : \"/memobase/test_record_set_6/Export_Bilder_der_Arbeit_8.csv\"}",
Jonas Waeber's avatar
Jonas Waeber committed
213 214 215 216
            expectedReportValue = Report(
                id = "Export_Bilder_der_Arbeit_8.csv",
                status = "FAILURE",
                message = "CSV ERROR: Fields num seems to be 1 on each row, but on 2th csv row, fields num is 5. for file /memobase/test_record_set_6/Export_Bilder_der_Arbeit_8.csv."
Jonas Waeber's avatar
Jonas Waeber committed
217
            ),
Jonas Waeber's avatar
Jonas Waeber committed
218
            expectedProcessReport = ProcessReport(
Jonas Waeber's avatar
Jonas Waeber committed
219 220
                id = "text-file-validation",
                status = "FAILURE",
Jonas Waeber's avatar
Jonas Waeber committed
221 222 223
                total = 1,
                failures = 1,
                successes = 0
Jonas Waeber's avatar
Jonas Waeber committed
224
            )
225 226 227 228 229
        ),
        TestParams(
            "test7.yml",
            expectedKey = "valid_xml.xml",
            expectedValue = "{\"format\" : \"XML\", \"path\" : \"/memobase/test_record_set_7/valid_xml.xml\"}",
Jonas Waeber's avatar
Jonas Waeber committed
230 231 232 233
            expectedReportValue = Report(
                id = "valid_xml.xml",
                status = "SUCCESS",
                message = "Validated file at path /memobase/test_record_set_7/valid_xml.xml with format XML."
234
            ),
Jonas Waeber's avatar
Jonas Waeber committed
235
            expectedProcessReport = ProcessReport(
236 237
                id = "text-file-validation",
                status = "SUCCESS",
Jonas Waeber's avatar
Jonas Waeber committed
238 239 240
                total = 1,
                failures = 0,
                successes = 1
241 242 243 244 245 246
            )
        ),
        TestParams(
            "test8.yml",
            expectedKey = "invalid.xml",
            expectedValue = "{\"format\" : \"ERROR\", \"path\" : \"/memobase/test_record_set_8/invalid.xml\"}",
Jonas Waeber's avatar
Jonas Waeber committed
247 248 249 250
            expectedReportValue = Report(
                id = "invalid.xml",
                status = "FAILURE",
                message = "XML ERROR: Element type \"foxml:objectProperties\" must be followed by either attribute specifications, \">\" or \"/>\". for file /memobase/test_record_set_8/invalid.xml."
251
            ),
Jonas Waeber's avatar
Jonas Waeber committed
252
            expectedProcessReport = ProcessReport(
253 254
                id = "text-file-validation",
                status = "FAILURE",
Jonas Waeber's avatar
Jonas Waeber committed
255 256 257
                total = 1,
                failures = 1,
                successes = 0
258
            )
259
        )
Jonas Waeber's avatar
Jonas Waeber committed
260
    )
Jonas Waeber's avatar
Jonas Waeber committed
261
}