Due to a scheduled upgrade to version 14.10, GitLab will be unavailabe on Monday 30.05., from 19:00 until 20:00.

IntegrationTests.kt 9.28 KB
Newer Older
Jonas Waeber's avatar
Jonas Waeber committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
/*
 * Copyright (C) 2019  Memobase
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
package org.memobase

Jonas Waeber's avatar
Jonas Waeber committed
19
20
import ch.memobase.rdf.NS
import ch.memobase.reporting.Report
Jonas Waeber's avatar
Jonas Waeber committed
21
import ch.memobase.reporting.ReportStatus
22
import com.beust.klaxon.Klaxon
23
24
25
26
import java.io.File
import java.io.FileOutputStream
import java.nio.charset.Charset
import java.util.stream.Stream
Jonas Waeber's avatar
Jonas Waeber committed
27
28
29
30
31
32
import org.apache.jena.rdf.model.ModelFactory
import org.apache.jena.riot.Lang
import org.apache.jena.riot.RDFDataMgr
import org.apache.jena.riot.RDFFormat
import org.apache.kafka.common.header.internals.RecordHeader
import org.apache.kafka.common.header.internals.RecordHeaders
Jonas Waeber's avatar
Jonas Waeber committed
33
34
35
36
37
import org.apache.kafka.common.serialization.StringDeserializer
import org.apache.kafka.common.serialization.StringSerializer
import org.apache.kafka.streams.TopologyTestDriver
import org.apache.kafka.streams.test.ConsumerRecordFactory
import org.assertj.core.api.Assertions.assertThat
Jonas Waeber's avatar
Jonas Waeber committed
38
import org.junit.jupiter.api.TestInstance
39
import org.junit.jupiter.api.assertAll
Jonas Waeber's avatar
Jonas Waeber committed
40
41
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.MethodSource
Jonas Waeber's avatar
Jonas Waeber committed
42
import org.memobase.params.IntegrationTestParams
Jonas Waeber's avatar
Jonas Waeber committed
43
44

@TestInstance(TestInstance.Lifecycle.PER_CLASS)
Jonas Waeber's avatar
Jonas Waeber committed
45
class IntegrationTests {
Jonas Waeber's avatar
Jonas Waeber committed
46

Jonas Waeber's avatar
Jonas Waeber committed
47
48
    private val klaxon = Klaxon()

Jonas Waeber's avatar
Jonas Waeber committed
49
    private val resourcePath = "src/test/resources"
Jonas Waeber's avatar
Jonas Waeber committed
50
51
52
    private fun readFile(fileName: String): String {
        return File("$resourcePath/$fileName").readText(Charset.defaultCharset())
    }
Jonas Waeber's avatar
Jonas Waeber committed
53

Jonas Waeber's avatar
Jonas Waeber committed
54
    private val regex = Regex("(_:B[A-Za-z0-9]+)")
Jonas Waeber's avatar
Jonas Waeber committed
55
56
57
58
59
60
61
62
63
64
65
66
67
68
    private val regexTime = Regex("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}")

    private fun sort(source: List<String>): String {
        return source.map {
            var replacedString = it
            for (matchResult in regex.findAll(it)) {
                replacedString = replacedString.replace(matchResult.groups[0]?.value.orEmpty(), "_:B")
            }
            for (matchResult in regexTime.findAll(it)) {
                replacedString = replacedString.replace(matchResult.groups[0]?.value.orEmpty(), "2020-10-10T09:10:22")
            }
            replacedString
        }.sorted().reduce { acc, s -> acc + "\n" + s }.trim()
    }
Jonas Waeber's avatar
Jonas Waeber committed
69
70
71

    @ParameterizedTest
    @MethodSource("kafkaTests")
Jonas Waeber's avatar
Jonas Waeber committed
72
    fun `test kafka topology`(params: IntegrationTestParams) {
73
        val service = Service("kafkaTest${params.count}.yml")
Jonas Waeber's avatar
Jonas Waeber committed
74
        val output = readFile("kafkaTests/${params.count}/output.nt").lines()
75
        val settings = service.settings
Jonas Waeber's avatar
Jonas Waeber committed
76
77

        val headers = RecordHeaders()
Jonas Waeber's avatar
Jonas Waeber committed
78
79
80
        headers.add(RecordHeader("sessionId", "1".toByteArray()))
        headers.add(RecordHeader("recordSetId", "ati-002".toByteArray()))
        headers.add(RecordHeader("institutionId", "ati".toByteArray()))
Jonas Waeber's avatar
Jonas Waeber committed
81
82
83
84
85
86
87
88
        headers.add(RecordHeader("isPublished", "false".toByteArray()))
        headers.add(RecordHeader("xmlRecordTag", "record".toByteArray()))
        headers.add(RecordHeader("xmlIdentifierFieldName", "identifierMain".toByteArray()))
        headers.add(RecordHeader("tableSheetIndex", "1".toByteArray()))
        headers.add(RecordHeader("tableHeaderCount", "1".toByteArray()))
        headers.add(RecordHeader("tableHeaderIndex", "1".toByteArray()))
        headers.add(RecordHeader("tableIdentifierIndex", "1".toByteArray()))

Jonas Waeber's avatar
Jonas Waeber committed
89
        val testDriver = TopologyTestDriver(KafkaTopology(settings).prepare().build(), settings.kafkaStreamsSettings)
Jonas Waeber's avatar
Jonas Waeber committed
90
91
92
        val factory = ConsumerRecordFactory(
            StringSerializer(), StringSerializer()
        )
Jonas Waeber's avatar
Jonas Waeber committed
93
94
95

        testDriver.pipeInput(
            factory.create(
Jonas Waeber's avatar
Jonas Waeber committed
96
                "mb-di-config-test",
Jonas Waeber's avatar
Jonas Waeber committed
97
                "ati-002#mapping",
Jonas Waeber's avatar
Jonas Waeber committed
98
99
100
101
                readFile("kafkaTests/${params.count}/config/mapping.yml")
            )
        )

Jonas Waeber's avatar
Jonas Waeber committed
102
103
        testDriver.pipeInput(
            factory.create(
Jonas Waeber's avatar
Jonas Waeber committed
104
                settings.inputTopic, params.key, readFile("kafkaTests/${params.count}/input.json"), headers
Jonas Waeber's avatar
Jonas Waeber committed
105
106
107
            )
        )

Jonas Waeber's avatar
Jonas Waeber committed
108
109
110
111
112
113
114
115
116
117
118
119
        val reportRecord = testDriver.readOutput(
            service.settings.processReportTopic,
            StringDeserializer(),
            StringDeserializer()
        )
        val reportString = reportRecord.value()
        val report = klaxon.parse<Report>(reportString)!!
        if (report.status == ReportStatus.fatal) {
            assertThat(report)
                .isEqualTo(params.report)
            return
        }
Jonas Waeber's avatar
Jonas Waeber committed
120
121

        val record = testDriver.readOutput(
Jonas Waeber's avatar
Jonas Waeber committed
122
123
124
125
126
            settings.outputTopic,
            StringDeserializer(),
            StringDeserializer()
        )

Jonas Waeber's avatar
Jonas Waeber committed
127
128
129
130
131
132
133
134
135
136
        val model = ModelFactory.createDefaultModel()
        NS.prefixMapping.map {
            model.setNsPrefix(it.key, it.value)
        }
        RDFDataMgr.read(model, record.value().byteInputStream(), Lang.NTRIPLES)
        RDFDataMgr.write(
            FileOutputStream("$resourcePath/kafkaTests/${params.count}/turtle-output.ttl"),
            model,
            RDFFormat.TURTLE_PRETTY
        )
Jonas Waeber's avatar
Jonas Waeber committed
137

138
        val key = record.key()
Jonas Waeber's avatar
Jonas Waeber committed
139
        val sortedResult = sort(record.value().lines()).trim()
Jonas Waeber's avatar
Jonas Waeber committed
140

141
142
143
144
145
146
147
148
149
150
151
152
        assertAll("",
            { assertThat(key).isEqualTo(params.expectedKey) },
            {
                assertThat(sortedResult)
                    .isEqualTo(sort(output).trim())
            },
            {
                assertThat(Klaxon().parse<Report>(reportString))
                    .isEqualTo(params.report)
            }

        )
Jonas Waeber's avatar
Jonas Waeber committed
153
    }
Jonas Waeber's avatar
Jonas Waeber committed
154

Jonas Waeber's avatar
Jonas Waeber committed
155
    private fun kafkaTests() = Stream.of(
Jonas Waeber's avatar
Jonas Waeber committed
156
        IntegrationTestParams(
157
158
159
160
161
162
163
164
165
166
167
            1,
            "MEI_49884",
            "https://memobase.ch/record/test-record-set-id-MEI_49884",
            Report(
                "MEI_49884",
                "FATAL",
                "No Record Id Found: Found multiple values in the field 'identifierOriginal' for identifiers: 22861, 22861, 22861.",
                Service.step
            )

        )/*,
Jonas Waeber's avatar
Jonas Waeber committed
168
        IntegrationTestParams(
Jonas Waeber's avatar
Jonas Waeber committed
169
170
            2,
            "Sig Han 1293",
Jonas Waeber's avatar
Jonas Waeber committed
171
            "https://memobase.ch/record/test-record-set-id-Sig_Han_1293",
Jonas Waeber's avatar
Jonas Waeber committed
172
            Report(
Jonas Waeber's avatar
Jonas Waeber committed
173
                "https://memobase.ch/record/test-record-set-id-Sig_Han_1293",
Jonas Waeber's avatar
Jonas Waeber committed
174
                "SUCCESS",
Jonas Waeber's avatar
Jonas Waeber committed
175
                "Successfully mapped record with id https://memobase.ch/record/test-record-set-id-Sig_Han_1293."
176
            )
177
        ),
Jonas Waeber's avatar
Jonas Waeber committed
178
        IntegrationTestParams(
179
180
            3,
            "Sig Han 1293",
181
            "ttps://memobase.ch/record/TEST_RECORD_SET-Sig_Han_1293",
Jonas Waeber's avatar
Jonas Waeber committed
182
            Report(
Jonas Waeber's avatar
Jonas Waeber committed
183
                "https://memobase.ch/record/test-record-set-id-Sig_Han_1293",
Jonas Waeber's avatar
Jonas Waeber committed
184
                "SUCCESS",
Jonas Waeber's avatar
Jonas Waeber committed
185
                "Successfully mapped record with id https://memobase.ch/record/test-record-set-id-Sig_Han_1293."
186
            )
187
        ),
Jonas Waeber's avatar
Jonas Waeber committed
188
        IntegrationTestParams(
189
            4,
190
            "InputKey",
Jonas Waeber's avatar
Jonas Waeber committed
191
           "https://memobase.ch/record/test-record-set-id-sigantur-example",
Jonas Waeber's avatar
Jonas Waeber committed
192
            Report(
Jonas Waeber's avatar
Jonas Waeber committed
193
                "https://memobase.ch/record/test-record-set-id-sigantur-example",
Jonas Waeber's avatar
Jonas Waeber committed
194
                "SUCCESS",
Jonas Waeber's avatar
Jonas Waeber committed
195
                "Successfully mapped record with id https://memobase.ch/record/test-record-set-id-sigantur-example."
196
            )
Jonas Waeber's avatar
Jonas Waeber committed
197
        ),
Jonas Waeber's avatar
Jonas Waeber committed
198
        IntegrationTestParams(
199
200
            5,
            "ID_1",
Jonas Waeber's avatar
Jonas Waeber committed
201
            "https://memobase.ch/record/test-record-set-id-ID_1",
Jonas Waeber's avatar
Jonas Waeber committed
202
            Report(
Jonas Waeber's avatar
Jonas Waeber committed
203
                "https://memobase.ch/record/test-record-set-id-ID_1",
Jonas Waeber's avatar
Jonas Waeber committed
204
                "SUCCESS",
Jonas Waeber's avatar
Jonas Waeber committed
205
                "",
Jonas Waeber's avatar
Jonas Waeber committed
206
                Service.step
207
            )
Jonas Waeber's avatar
Jonas Waeber committed
208
        ),
Jonas Waeber's avatar
Jonas Waeber committed
209
        IntegrationTestParams(
Jonas Waeber's avatar
Jonas Waeber committed
210
211
            6,
            "ID_1",
Jonas Waeber's avatar
Jonas Waeber committed
212
           "https://memobase.ch/record/test-record-set-id-ID_1",
Jonas Waeber's avatar
Jonas Waeber committed
213
            Report(
Jonas Waeber's avatar
Jonas Waeber committed
214
                "https://memobase.ch/record/test-record-set-id-ID_1",
Jonas Waeber's avatar
Jonas Waeber committed
215
                "SUCCESS",
Jonas Waeber's avatar
Jonas Waeber committed
216
                "Successfully mapped record with id https://memobase.ch/record/test-record-set-id-ID_1."
Jonas Waeber's avatar
Jonas Waeber committed
217
            )
Jonas Waeber's avatar
Jonas Waeber committed
218
        ),
Jonas Waeber's avatar
Jonas Waeber committed
219
        IntegrationTestParams(
Jonas Waeber's avatar
Jonas Waeber committed
220
221
            7,
            "ID_1",
Jonas Waeber's avatar
Jonas Waeber committed
222
           "https://memobase.ch/record/test-record-set-id-ID_1",
Jonas Waeber's avatar
Jonas Waeber committed
223
            Report(
Jonas Waeber's avatar
Jonas Waeber committed
224
                "https://memobase.ch/record/test-record-set-id-ID_1",
Jonas Waeber's avatar
Jonas Waeber committed
225
                "SUCCESS",
Jonas Waeber's avatar
Jonas Waeber committed
226
                "Successfully mapped record with id https://memobase.ch/record/test-record-set-id-ID_1."
Jonas Waeber's avatar
Jonas Waeber committed
227
            )
228
        ),
Jonas Waeber's avatar
Jonas Waeber committed
229
        IntegrationTestParams(
230
231
            8,
            "ID_1",
Jonas Waeber's avatar
Jonas Waeber committed
232
           "https://memobase.ch/record/test-record-set-id-ID_1",
Jonas Waeber's avatar
Jonas Waeber committed
233
            Report(
Jonas Waeber's avatar
Jonas Waeber committed
234
                "https://memobase.ch/record/test-record-set-id-ID_1",
Jonas Waeber's avatar
Jonas Waeber committed
235
                "SUCCESS",
Jonas Waeber's avatar
Jonas Waeber committed
236
                "Successfully mapped record with id https://memobase.ch/record/test-record-set-id-ID_1."
237
            )
Jonas Waeber's avatar
Jonas Waeber committed
238
        ),
Jonas Waeber's avatar
Jonas Waeber committed
239
        IntegrationTestParams(
Jonas Waeber's avatar
Jonas Waeber committed
240
241
            9,
            "ID_1",
Jonas Waeber's avatar
Jonas Waeber committed
242
           "https://memobase.ch/record/test-record-set-id-ID_1",
Jonas Waeber's avatar
Jonas Waeber committed
243
            Report(
Jonas Waeber's avatar
Jonas Waeber committed
244
                "https://memobase.ch/record/test-record-set-id-ID_1",
Jonas Waeber's avatar
Jonas Waeber committed
245
                "SUCCESS",
Jonas Waeber's avatar
Jonas Waeber committed
246
                "Successfully mapped record with id https://memobase.ch/record/test-record-set-id-ID_1."
Jonas Waeber's avatar
Jonas Waeber committed
247
            )
248
        )*/
Jonas Waeber's avatar
Jonas Waeber committed
249
    )
Jonas Waeber's avatar
Jonas Waeber committed
250
}