Tests.kt 10.9 KB
Newer Older
Jonas Waeber's avatar
Jonas Waeber committed
1
/*
Jonas Waeber's avatar
Jonas Waeber committed
2
 * sftp-reader
Jonas Waeber's avatar
Jonas Waeber committed
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
 * Copyright (C) 2019  Memobase
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
18
package org.memobase
Jonas Waeber's avatar
Jonas Waeber committed
19

Jonas Waeber's avatar
Jonas Waeber committed
20
21
import com.beust.klaxon.Klaxon
import java.io.FileInputStream
Jonas Waeber's avatar
Jonas Waeber committed
22
import java.nio.file.Paths
Jonas Waeber's avatar
Jonas Waeber committed
23
24
25
26
import java.time.Duration
import java.util.Properties
import java.util.stream.Stream
import org.apache.kafka.clients.consumer.ConsumerConfig
Jonas Waeber's avatar
Jonas Waeber committed
27
import org.apache.kafka.clients.consumer.ConsumerRecord
Jonas Waeber's avatar
Jonas Waeber committed
28
29
import org.apache.kafka.clients.consumer.KafkaConsumer
import org.apache.kafka.common.serialization.StringDeserializer
Jonas Waeber's avatar
Jonas Waeber committed
30
import org.apache.logging.log4j.LogManager
Jonas Waeber's avatar
Jonas Waeber committed
31
import org.assertj.core.api.Assertions.assertThat
Jonas Waeber's avatar
Jonas Waeber committed
32
import org.junit.jupiter.api.TestInstance
33
import org.junit.jupiter.api.extension.ExtendWith
Jonas Waeber's avatar
Jonas Waeber committed
34
35
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.MethodSource
Jonas Waeber's avatar
Jonas Waeber committed
36
37
import org.memobase.testing.EmbeddedKafkaExtension
import org.memobase.testing.EmbeddedSftpServer
Jonas Waeber's avatar
Jonas Waeber committed
38

Jonas Waeber's avatar
Jonas Waeber committed
39
@ExtendWith(EmbeddedKafkaExtension::class)
Jonas Waeber's avatar
Jonas Waeber committed
40
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
Jonas Waeber's avatar
Jonas Waeber committed
41
class Tests {
Jonas Waeber's avatar
Jonas Waeber committed
42
    private val log = LogManager.getLogger("LocalTestsLogger")
Jonas Waeber's avatar
Jonas Waeber committed
43

Jonas Waeber's avatar
Jonas Waeber committed
44
45
46
    private val sftpServer = EmbeddedSftpServer(22000, "user", "password")

    init {
Jonas Waeber's avatar
Jonas Waeber committed
47
48
49
50
51
        val files = listOf(
            Pair("/memobase/test_record_set_1", "brandt.csv"),
            Pair("/memobase/test_record_set_2", "bauGAZH_metadaten.csv"),
            Pair("/memobase/test_record_set_3", "invalid.csv"),
            Pair("/memobase/test_record_set_4", "file.txt"),
Jonas Waeber's avatar
Jonas Waeber committed
52
            Pair("/memobase/test_record_set_5", "20190906_Brandt_Metadaten.xlsx"),
53
54
55
            Pair("/memobase/test_record_set_6", "Export_Bilder_der_Arbeit_8.csv"),
            Pair("/memobase/test_record_set_7", "valid_xml.xml"),
            Pair("/memobase/test_record_set_8", "invalid.xml")
Jonas Waeber's avatar
Jonas Waeber committed
56
        )
Jonas Waeber's avatar
Jonas Waeber committed
57
58
59
60

        for (pair in files) {
            sftpServer.putFile(Paths.get(pair.first, pair.second).toString(), FileInputStream(Paths.get("src/test/resources/data", pair.second).toFile()))
        }
Jonas Waeber's avatar
Jonas Waeber committed
61
62
63
64
65
66
67
68
69
70
71
72
73
    }

    private val consumer: KafkaConsumer<String, String>

    init {
        val props = Properties()
        props.setProperty(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:12345")
        props.setProperty(ConsumerConfig.CLIENT_ID_CONFIG, "test-group-1")
        props.setProperty(ConsumerConfig.GROUP_ID_CONFIG, "test-group-1")
        props.setProperty(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer::class.qualifiedName)
        props.setProperty(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer::class.qualifiedName)
        props.setProperty(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest")
        consumer = KafkaConsumer(props)
Jonas Waeber's avatar
Jonas Waeber committed
74
        consumer.subscribe(listOf("sftp-reader-p1-j1", "sftp-reader-p1-j1-reporting", "p1-reporting"))
Jonas Waeber's avatar
Jonas Waeber committed
75
76
77
78
    }

    @ParameterizedTest
    @MethodSource("directoryReaderTests")
Jonas Waeber's avatar
Jonas Waeber committed
79
    fun `read files from sftp`(params: TestParams) {
80
81
        val service = Service(params.configFile)
        service.run()
Jonas Waeber's avatar
Jonas Waeber committed
82

83
84
85
        val topic = service.settings.outputTopic
        val reportingTopic = service.settings.outputTopic + "-reporting"
        val processReportingTopic = service.settings.processReportTopic
Jonas Waeber's avatar
Jonas Waeber committed
86

Jonas Waeber's avatar
Jonas Waeber committed
87
88
        val totalConsumerRecords = mutableListOf<ConsumerRecord<String, String>>()

Jonas Waeber's avatar
Jonas Waeber committed
89
        var result = consumer.poll(Duration.ofMillis(10))
Jonas Waeber's avatar
Jonas Waeber committed
90
91
92
93
94
        while (totalConsumerRecords.size != 3) {
            if (result.count() > 0) {
                totalConsumerRecords.addAll(result.asIterable())
            }
            log.error(result.count())
Jonas Waeber's avatar
Jonas Waeber committed
95
96
97
            result = consumer.poll(Duration.ofMillis(10))
        }

Jonas Waeber's avatar
Jonas Waeber committed
98
        assertThat(totalConsumerRecords.find { value -> value.topic() == topic })
99
100
101
            .describedAs("Message Test")
            .hasFieldOrPropertyWithValue("key", params.expectedKey)
            .hasFieldOrPropertyWithValue("value", params.expectedValue)
Jonas Waeber's avatar
Jonas Waeber committed
102

Jonas Waeber's avatar
Jonas Waeber committed
103
        assertThat(totalConsumerRecords.find { value -> value.topic() == reportingTopic })
104
105
106
            .describedAs("Report Test")
            .hasFieldOrPropertyWithValue("key", params.expectedKey)
            .hasFieldOrPropertyWithValue("value", params.expectedReportValue)
Jonas Waeber's avatar
Jonas Waeber committed
107

Jonas Waeber's avatar
Jonas Waeber committed
108
        assertThat(totalConsumerRecords.find { value -> value.topic() == processReportingTopic })
109
110
111
            .describedAs("Process Report Test")
            .hasFieldOrPropertyWithValue("key", params.expectedProcessReport.id)
            .hasFieldOrPropertyWithValue("value", Klaxon().toJsonString(params.expectedProcessReport))
Jonas Waeber's avatar
Jonas Waeber committed
112
    }
Jonas Waeber's avatar
Jonas Waeber committed
113
114

    private fun directoryReaderTests() = Stream.of(
115
116
117
        TestParams(
            "test1.yml",
            expectedKey = "brandt.csv",
Jonas Waeber's avatar
Jonas Waeber committed
118
            expectedValue = "{\"format\" : \"CSV\", \"path\" : \"/memobase/test_record_set_1/brandt.csv\"}",
119
120
121
122
            expectedReportValue = Klaxon().toJsonString(
                Report(
                    id = "brandt.csv",
                    status = "SUCCESS",
Jonas Waeber's avatar
Jonas Waeber committed
123
                    message = "Validated file at path /memobase/test_record_set_1/brandt.csv with format CSV."
124
                )
Jonas Waeber's avatar
Jonas Waeber committed
125
            ),
126
            expectedProcessReport = Report(
Jonas Waeber's avatar
Jonas Waeber committed
127
                id = "text-file-validation",
128
129
130
131
132
133
134
                status = "SUCCESS",
                message = "Successfully validated 1 files."
            )
        ),
        TestParams(
            "test2.yml",
            expectedKey = "bauGAZH_metadaten.csv",
Jonas Waeber's avatar
Jonas Waeber committed
135
            expectedValue = "{\"format\" : \"CSV\", \"path\" : \"/memobase/test_record_set_2/bauGAZH_metadaten.csv\"}",
136
137
138
139
            expectedReportValue = Klaxon().toJsonString(
                Report(
                    id = "bauGAZH_metadaten.csv",
                    status = "SUCCESS",
Jonas Waeber's avatar
Jonas Waeber committed
140
                    message = "Validated file at path /memobase/test_record_set_2/bauGAZH_metadaten.csv with format CSV."
141
142
                )
            ),
143
            expectedProcessReport = Report(
Jonas Waeber's avatar
Jonas Waeber committed
144
                id = "text-file-validation",
145
146
147
148
149
150
151
                status = "SUCCESS",
                message = "Successfully validated 1 files."
            )
        ),
        TestParams(
            "test3.yml",
            expectedKey = "invalid.csv",
Jonas Waeber's avatar
Jonas Waeber committed
152
            expectedValue = "{\"format\" : \"ERROR\", \"path\" : \"/memobase/test_record_set_3/invalid.csv\"}",
153
154
155
            expectedReportValue = Klaxon().toJsonString(
                Report(
                    id = "invalid.csv",
156
                    status = "FAILURE",
Jonas Waeber's avatar
Jonas Waeber committed
157
                    message = "CSV ERROR: Fields num seems to be 4 on each row, but on 2th csv row, fields num is 6. for file /memobase/test_record_set_3/invalid.csv."
158
159
                )
            ),
160
            expectedProcessReport = Report(
Jonas Waeber's avatar
Jonas Waeber committed
161
                id = "text-file-validation",
162
163
164
165
166
167
168
                status = "FAILURE",
                message = "Failed to validate 1 of 1 files."
            )
        ),
        TestParams(
            "test4.yml",
            expectedKey = "file.txt",
Jonas Waeber's avatar
Jonas Waeber committed
169
            expectedValue = "{\"format\" : \"ERROR\", \"path\" : \"/memobase/test_record_set_4/file.txt\"}",
170
171
172
            expectedReportValue = Klaxon().toJsonString(
                Report(
                    id = "file.txt",
173
                    status = "FAILURE",
174
                    message = "File Extension Error: Not a valid file extension: file.txt."
175
                )
176
177
            ),
            expectedProcessReport = Report(
Jonas Waeber's avatar
Jonas Waeber committed
178
                id = "text-file-validation",
179
180
                status = "FAILURE",
                message = "Failed to validate 1 of 1 files."
Jonas Waeber's avatar
Jonas Waeber committed
181
            )
182
183
184
        ),
        TestParams(
            "test5.yml",
Jonas Waeber's avatar
Jonas Waeber committed
185
186
            expectedKey = "20190906_Brandt_Metadaten.xlsx",
            expectedValue = "{\"format\" : \"XLSX\", \"path\" : \"/memobase/test_record_set_5/20190906_Brandt_Metadaten.xlsx\"}",
187
188
            expectedReportValue = Klaxon().toJsonString(
                Report(
Jonas Waeber's avatar
Jonas Waeber committed
189
                    id = "20190906_Brandt_Metadaten.xlsx",
190
                    status = "SUCCESS",
Jonas Waeber's avatar
Jonas Waeber committed
191
                    message = "Validated file at path /memobase/test_record_set_5/20190906_Brandt_Metadaten.xlsx with format XLSX."
192
193
194
                )
            ),
            expectedProcessReport = Report(
Jonas Waeber's avatar
Jonas Waeber committed
195
                id = "text-file-validation",
196
197
198
                status = "SUCCESS",
                message = "Successfully validated 1 files."
            )
Jonas Waeber's avatar
Jonas Waeber committed
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
        ),
        TestParams(
            "test6.yml",
            expectedKey = "Export_Bilder_der_Arbeit_8.csv",
            expectedValue = "{\"format\" : \"ERROR\", \"path\" : \"/memobase/test_record_set_6/Export_Bilder_der_Arbeit_8.csv\"}",
            expectedReportValue = Klaxon().toJsonString(
                Report(
                    id = "Export_Bilder_der_Arbeit_8.csv",
                    status = "FAILURE",
                    message = "CSV ERROR: Fields num seems to be 1 on each row, but on 2th csv row, fields num is 5. for file /memobase/test_record_set_6/Export_Bilder_der_Arbeit_8.csv."
                )
            ),
            expectedProcessReport = Report(
                id = "text-file-validation",
                status = "FAILURE",
                message = "Failed to validate 1 of 1 files."
            )
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
        ),
        TestParams(
            "test7.yml",
            expectedKey = "valid_xml.xml",
            expectedValue = "{\"format\" : \"XML\", \"path\" : \"/memobase/test_record_set_7/valid_xml.xml\"}",
            expectedReportValue = Klaxon().toJsonString(
                Report(
                    id = "valid_xml.xml",
                    status = "SUCCESS",
                    message = "Validated file at path /memobase/test_record_set_7/valid_xml.xml with format XML."
                )
            ),
            expectedProcessReport = Report(
                id = "text-file-validation",
                status = "SUCCESS",
                message = "Successfully validated 1 files."
            )
        ),
        TestParams(
            "test8.yml",
            expectedKey = "invalid.xml",
            expectedValue = "{\"format\" : \"ERROR\", \"path\" : \"/memobase/test_record_set_8/invalid.xml\"}",
            expectedReportValue = Klaxon().toJsonString(
                Report(
                    id = "invalid.xml",
                    status = "FAILURE",
                    message = "XML ERROR: Element type \"foxml:objectProperties\" must be followed by either attribute specifications, \">\" or \"/>\". for file /memobase/test_record_set_8/invalid.xml."
                )
            ),
            expectedProcessReport = Report(
                id = "text-file-validation",
                status = "FAILURE",
                message = "Failed to validate 1 of 1 files."
            )
250
        )
Jonas Waeber's avatar
Jonas Waeber committed
251
    )
Jonas Waeber's avatar
Jonas Waeber committed
252
}