Tests.kt 10.9 KB
Newer Older
Jonas Waeber's avatar
Jonas Waeber committed
1
/*
Jonas Waeber's avatar
Jonas Waeber committed
2
 * sftp-reader
Jonas Waeber's avatar
Jonas Waeber committed
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17
 * Copyright (C) 2019  Memobase
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
18
package org.memobase
Jonas Waeber's avatar
Jonas Waeber committed
19

Jonas Waeber's avatar
Jonas Waeber committed
20 21
import com.beust.klaxon.Klaxon
import java.io.FileInputStream
Jonas Waeber's avatar
Jonas Waeber committed
22
import java.nio.file.Paths
Jonas Waeber's avatar
Jonas Waeber committed
23 24 25 26
import java.time.Duration
import java.util.Properties
import java.util.stream.Stream
import org.apache.kafka.clients.consumer.ConsumerConfig
Jonas Waeber's avatar
Jonas Waeber committed
27
import org.apache.kafka.clients.consumer.ConsumerRecord
Jonas Waeber's avatar
Jonas Waeber committed
28 29
import org.apache.kafka.clients.consumer.KafkaConsumer
import org.apache.kafka.common.serialization.StringDeserializer
Jonas Waeber's avatar
Jonas Waeber committed
30
import org.apache.logging.log4j.LogManager
Jonas Waeber's avatar
Jonas Waeber committed
31
import org.assertj.core.api.Assertions.assertThat
Jonas Waeber's avatar
Jonas Waeber committed
32
import org.junit.jupiter.api.TestInstance
33
import org.junit.jupiter.api.extension.ExtendWith
Jonas Waeber's avatar
Jonas Waeber committed
34 35
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.MethodSource
Jonas Waeber's avatar
Jonas Waeber committed
36 37
import org.memobase.testing.EmbeddedKafkaExtension
import org.memobase.testing.EmbeddedSftpServer
Jonas Waeber's avatar
Jonas Waeber committed
38

Jonas Waeber's avatar
Jonas Waeber committed
39
@ExtendWith(EmbeddedKafkaExtension::class)
Jonas Waeber's avatar
Jonas Waeber committed
40
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
Jonas Waeber's avatar
Jonas Waeber committed
41
class Tests {
Jonas Waeber's avatar
Jonas Waeber committed
42
    private val log = LogManager.getLogger("LocalTestsLogger")
Jonas Waeber's avatar
Jonas Waeber committed
43

Jonas Waeber's avatar
Jonas Waeber committed
44 45 46
    private val sftpServer = EmbeddedSftpServer(22000, "user", "password")

    init {
Jonas Waeber's avatar
Jonas Waeber committed
47 48 49 50 51
        val files = listOf(
            Pair("/memobase/test_record_set_1", "brandt.csv"),
            Pair("/memobase/test_record_set_2", "bauGAZH_metadaten.csv"),
            Pair("/memobase/test_record_set_3", "invalid.csv"),
            Pair("/memobase/test_record_set_4", "file.txt"),
Jonas Waeber's avatar
Jonas Waeber committed
52
            Pair("/memobase/test_record_set_5", "20190906_Brandt_Metadaten.xlsx"),
53 54 55
            Pair("/memobase/test_record_set_6", "Export_Bilder_der_Arbeit_8.csv"),
            Pair("/memobase/test_record_set_7", "valid_xml.xml"),
            Pair("/memobase/test_record_set_8", "invalid.xml")
Jonas Waeber's avatar
Jonas Waeber committed
56
        )
Jonas Waeber's avatar
Jonas Waeber committed
57 58 59 60

        for (pair in files) {
            sftpServer.putFile(Paths.get(pair.first, pair.second).toString(), FileInputStream(Paths.get("src/test/resources/data", pair.second).toFile()))
        }
Jonas Waeber's avatar
Jonas Waeber committed
61 62 63 64 65 66 67 68 69 70 71 72 73
    }

    private val consumer: KafkaConsumer<String, String>

    init {
        val props = Properties()
        props.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:12345")
        props.setProperty(ConsumerConfig.CLIENT_ID_CONFIG, "test-group-1")
        props.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "test-group-1")
        props.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer::class.qualifiedName)
        props.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer::class.qualifiedName)
        props.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest")
        consumer = KafkaConsumer(props)
Jonas Waeber's avatar
Jonas Waeber committed
74
        consumer.subscribe(listOf("sftp-reader-p1-j1", "sftp-reader-p1-j1-reporting", "p1-reporting"))
Jonas Waeber's avatar
Jonas Waeber committed
75 76 77 78
    }

    @ParameterizedTest
    @MethodSource("directoryReaderTests")
Jonas Waeber's avatar
Jonas Waeber committed
79
    fun `read files from sftp`(params: TestParams) {
80 81
        val service = Service(params.configFile)
        service.run()
Jonas Waeber's avatar
Jonas Waeber committed
82

83 84 85
        val topic = service.settings.outputTopic
        val reportingTopic = service.settings.outputTopic + "-reporting"
        val processReportingTopic = service.settings.processReportTopic
Jonas Waeber's avatar
Jonas Waeber committed
86

Jonas Waeber's avatar
Jonas Waeber committed
87 88
        val totalConsumerRecords = mutableListOf<ConsumerRecord<String, String>>()

Jonas Waeber's avatar
Jonas Waeber committed
89
        var result = consumer.poll(Duration.ofMillis(10))
Jonas Waeber's avatar
Jonas Waeber committed
90 91 92 93 94
        while (totalConsumerRecords.size != 3) {
            if (result.count() > 0) {
                totalConsumerRecords.addAll(result.asIterable())
            }
            log.error(result.count())
Jonas Waeber's avatar
Jonas Waeber committed
95 96 97
            result = consumer.poll(Duration.ofMillis(10))
        }

Jonas Waeber's avatar
Jonas Waeber committed
98
        assertThat(totalConsumerRecords.find { value -> value.topic() == topic })
99 100 101
            .describedAs("Message Test")
            .hasFieldOrPropertyWithValue("key", params.expectedKey)
            .hasFieldOrPropertyWithValue("value", params.expectedValue)
Jonas Waeber's avatar
Jonas Waeber committed
102

Jonas Waeber's avatar
Jonas Waeber committed
103
        assertThat(totalConsumerRecords.find { value -> value.topic() == reportingTopic })
104 105 106
            .describedAs("Report Test")
            .hasFieldOrPropertyWithValue("key", params.expectedKey)
            .hasFieldOrPropertyWithValue("value", params.expectedReportValue)
Jonas Waeber's avatar
Jonas Waeber committed
107

Jonas Waeber's avatar
Jonas Waeber committed
108
        assertThat(totalConsumerRecords.find { value -> value.topic() == processReportingTopic })
109 110 111
            .describedAs("Process Report Test")
            .hasFieldOrPropertyWithValue("key", params.expectedProcessReport.id)
            .hasFieldOrPropertyWithValue("value", Klaxon().toJsonString(params.expectedProcessReport))
Jonas Waeber's avatar
Jonas Waeber committed
112
    }
Jonas Waeber's avatar
Jonas Waeber committed
113 114

    private fun directoryReaderTests() = Stream.of(
115 116 117
        TestParams(
            "test1.yml",
            expectedKey = "brandt.csv",
Jonas Waeber's avatar
Jonas Waeber committed
118
            expectedValue = "{\"format\" : \"CSV\", \"path\" : \"/memobase/test_record_set_1/brandt.csv\"}",
119 120 121 122
            expectedReportValue = Klaxon().toJsonString(
                Report(
                    id = "brandt.csv",
                    status = "SUCCESS",
Jonas Waeber's avatar
Jonas Waeber committed
123
                    message = "Validated file at path /memobase/test_record_set_1/brandt.csv with format CSV."
124
                )
Jonas Waeber's avatar
Jonas Waeber committed
125
            ),
126
            expectedProcessReport = Report(
Jonas Waeber's avatar
Jonas Waeber committed
127
                id = "text-file-validation",
128 129 130 131 132 133 134
                status = "SUCCESS",
                message = "Successfully validated 1 files."
            )
        ),
        TestParams(
            "test2.yml",
            expectedKey = "bauGAZH_metadaten.csv",
Jonas Waeber's avatar
Jonas Waeber committed
135
            expectedValue = "{\"format\" : \"CSV\", \"path\" : \"/memobase/test_record_set_2/bauGAZH_metadaten.csv\"}",
136 137 138 139
            expectedReportValue = Klaxon().toJsonString(
                Report(
                    id = "bauGAZH_metadaten.csv",
                    status = "SUCCESS",
Jonas Waeber's avatar
Jonas Waeber committed
140
                    message = "Validated file at path /memobase/test_record_set_2/bauGAZH_metadaten.csv with format CSV."
141 142
                )
            ),
143
            expectedProcessReport = Report(
Jonas Waeber's avatar
Jonas Waeber committed
144
                id = "text-file-validation",
145 146 147 148 149 150 151
                status = "SUCCESS",
                message = "Successfully validated 1 files."
            )
        ),
        TestParams(
            "test3.yml",
            expectedKey = "invalid.csv",
Jonas Waeber's avatar
Jonas Waeber committed
152
            expectedValue = "{\"format\" : \"ERROR\", \"path\" : \"/memobase/test_record_set_3/invalid.csv\"}",
153 154 155
            expectedReportValue = Klaxon().toJsonString(
                Report(
                    id = "invalid.csv",
156
                    status = "FAILURE",
Jonas Waeber's avatar
Jonas Waeber committed
157
                    message = "CSV ERROR: Fields num seems to be 4 on each row, but on 2th csv row, fields num is 6. for file /memobase/test_record_set_3/invalid.csv."
158 159
                )
            ),
160
            expectedProcessReport = Report(
Jonas Waeber's avatar
Jonas Waeber committed
161
                id = "text-file-validation",
162 163 164 165 166 167 168
                status = "FAILURE",
                message = "Failed to validate 1 of 1 files."
            )
        ),
        TestParams(
            "test4.yml",
            expectedKey = "file.txt",
Jonas Waeber's avatar
Jonas Waeber committed
169
            expectedValue = "{\"format\" : \"ERROR\", \"path\" : \"/memobase/test_record_set_4/file.txt\"}",
170 171 172
            expectedReportValue = Klaxon().toJsonString(
                Report(
                    id = "file.txt",
173
                    status = "FAILURE",
174
                    message = "File Extension Error: Not a valid file extension: file.txt."
175
                )
176 177
            ),
            expectedProcessReport = Report(
Jonas Waeber's avatar
Jonas Waeber committed
178
                id = "text-file-validation",
179 180
                status = "FAILURE",
                message = "Failed to validate 1 of 1 files."
Jonas Waeber's avatar
Jonas Waeber committed
181
            )
182 183 184
        ),
        TestParams(
            "test5.yml",
Jonas Waeber's avatar
Jonas Waeber committed
185 186
            expectedKey = "20190906_Brandt_Metadaten.xlsx",
            expectedValue = "{\"format\" : \"XLSX\", \"path\" : \"/memobase/test_record_set_5/20190906_Brandt_Metadaten.xlsx\"}",
187 188
            expectedReportValue = Klaxon().toJsonString(
                Report(
Jonas Waeber's avatar
Jonas Waeber committed
189
                    id = "20190906_Brandt_Metadaten.xlsx",
190
                    status = "SUCCESS",
Jonas Waeber's avatar
Jonas Waeber committed
191
                    message = "Validated file at path /memobase/test_record_set_5/20190906_Brandt_Metadaten.xlsx with format XLSX."
192 193 194
                )
            ),
            expectedProcessReport = Report(
Jonas Waeber's avatar
Jonas Waeber committed
195
                id = "text-file-validation",
196 197 198
                status = "SUCCESS",
                message = "Successfully validated 1 files."
            )
Jonas Waeber's avatar
Jonas Waeber committed
199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215
        ),
        TestParams(
            "test6.yml",
            expectedKey = "Export_Bilder_der_Arbeit_8.csv",
            expectedValue = "{\"format\" : \"ERROR\", \"path\" : \"/memobase/test_record_set_6/Export_Bilder_der_Arbeit_8.csv\"}",
            expectedReportValue = Klaxon().toJsonString(
                Report(
                    id = "Export_Bilder_der_Arbeit_8.csv",
                    status = "FAILURE",
                    message = "CSV ERROR: Fields num seems to be 1 on each row, but on 2th csv row, fields num is 5. for file /memobase/test_record_set_6/Export_Bilder_der_Arbeit_8.csv."
                )
            ),
            expectedProcessReport = Report(
                id = "text-file-validation",
                status = "FAILURE",
                message = "Failed to validate 1 of 1 files."
            )
216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249
        ),
        TestParams(
            "test7.yml",
            expectedKey = "valid_xml.xml",
            expectedValue = "{\"format\" : \"XML\", \"path\" : \"/memobase/test_record_set_7/valid_xml.xml\"}",
            expectedReportValue = Klaxon().toJsonString(
                Report(
                    id = "valid_xml.xml",
                    status = "SUCCESS",
                    message = "Validated file at path /memobase/test_record_set_7/valid_xml.xml with format XML."
                )
            ),
            expectedProcessReport = Report(
                id = "text-file-validation",
                status = "SUCCESS",
                message = "Successfully validated 1 files."
            )
        ),
        TestParams(
            "test8.yml",
            expectedKey = "invalid.xml",
            expectedValue = "{\"format\" : \"ERROR\", \"path\" : \"/memobase/test_record_set_8/invalid.xml\"}",
            expectedReportValue = Klaxon().toJsonString(
                Report(
                    id = "invalid.xml",
                    status = "FAILURE",
                    message = "XML ERROR: Element type \"foxml:objectProperties\" must be followed by either attribute specifications, \">\" or \"/>\". for file /memobase/test_record_set_8/invalid.xml."
                )
            ),
            expectedProcessReport = Report(
                id = "text-file-validation",
                status = "FAILURE",
                message = "Failed to validate 1 of 1 files."
            )
250
        )
Jonas Waeber's avatar
Jonas Waeber committed
251
    )
Jonas Waeber's avatar
Jonas Waeber committed
252
}