IntegrationTests.kt 9.07 KB
Newer Older
Jonas Waeber's avatar
Jonas Waeber committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
/*
 * Copyright (C) 2019  Memobase
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */
package org.memobase

Jonas Waeber's avatar
Jonas Waeber committed
19
20
import ch.memobase.rdf.NS
import ch.memobase.reporting.Report
21
import com.beust.klaxon.Klaxon
Jonas Waeber's avatar
Jonas Waeber committed
22
23
24
25
26
27
import org.apache.jena.rdf.model.ModelFactory
import org.apache.jena.riot.Lang
import org.apache.jena.riot.RDFDataMgr
import org.apache.jena.riot.RDFFormat
import org.apache.kafka.common.header.internals.RecordHeader
import org.apache.kafka.common.header.internals.RecordHeaders
Jonas Waeber's avatar
Jonas Waeber committed
28
29
30
31
32
import org.apache.kafka.common.serialization.StringDeserializer
import org.apache.kafka.common.serialization.StringSerializer
import org.apache.kafka.streams.TopologyTestDriver
import org.apache.kafka.streams.test.ConsumerRecordFactory
import org.assertj.core.api.Assertions.assertThat
Jonas Waeber's avatar
Jonas Waeber committed
33
import org.junit.jupiter.api.TestInstance
34
import org.junit.jupiter.api.assertAll
Jonas Waeber's avatar
Jonas Waeber committed
35
36
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.MethodSource
Jonas Waeber's avatar
Jonas Waeber committed
37
import org.memobase.params.IntegrationTestParams
Jonas Waeber's avatar
Jonas Waeber committed
38
import java.io.File
Jonas Waeber's avatar
Jonas Waeber committed
39
import java.io.FileOutputStream
Jonas Waeber's avatar
Jonas Waeber committed
40
import java.nio.charset.Charset
Jonas Waeber's avatar
Jonas Waeber committed
41
import java.util.stream.Stream
Jonas Waeber's avatar
Jonas Waeber committed
42
43

@TestInstance(TestInstance.Lifecycle.PER_CLASS)
Jonas Waeber's avatar
Jonas Waeber committed
44
class IntegrationTests {
Jonas Waeber's avatar
Jonas Waeber committed
45

Jonas Waeber's avatar
Jonas Waeber committed
46
    private val resourcePath = "src/test/resources"
Jonas Waeber's avatar
Jonas Waeber committed
47
48
49
    private fun readFile(fileName: String): String {
        return File("$resourcePath/$fileName").readText(Charset.defaultCharset())
    }
Jonas Waeber's avatar
Jonas Waeber committed
50

Jonas Waeber's avatar
Jonas Waeber committed
51
    private val regex = Regex("(_:B[A-Za-z0-9]+)")
Jonas Waeber's avatar
Jonas Waeber committed
52
53
54
55
56
57
58
59
60
61
62
63
64
65
    private val regexTime = Regex("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}")

    private fun sort(source: List<String>): String {
        return source.map {
            var replacedString = it
            for (matchResult in regex.findAll(it)) {
                replacedString = replacedString.replace(matchResult.groups[0]?.value.orEmpty(), "_:B")
            }
            for (matchResult in regexTime.findAll(it)) {
                replacedString = replacedString.replace(matchResult.groups[0]?.value.orEmpty(), "2020-10-10T09:10:22")
            }
            replacedString
        }.sorted().reduce { acc, s -> acc + "\n" + s }.trim()
    }
Jonas Waeber's avatar
Jonas Waeber committed
66
67
68

    @ParameterizedTest
    @MethodSource("kafkaTests")
Jonas Waeber's avatar
Jonas Waeber committed
69
    fun `test kafka topology`(params: IntegrationTestParams) {
70
        val service = Service("kafkaTest${params.count}.yml")
Jonas Waeber's avatar
Jonas Waeber committed
71
        val output = readFile("kafkaTests/${params.count}/output.nt").lines()
72
        val settings = service.settings
Jonas Waeber's avatar
Jonas Waeber committed
73
74
75
76
77
78
79
80
81
82
83
84
85
86

        val headers = RecordHeaders()
        headers.add(RecordHeader("sessionId", "test-session-id".toByteArray()))
        headers.add(RecordHeader("recordSetId", "test-record-set-id".toByteArray()))
        headers.add(RecordHeader("institutionId", "test-institution-id".toByteArray()))
        headers.add(RecordHeader("isPublished", "false".toByteArray()))
        headers.add(RecordHeader("xmlRecordTag", "record".toByteArray()))
        headers.add(RecordHeader("xmlIdentifierFieldName", "identifierMain".toByteArray()))
        headers.add(RecordHeader("tableSheetIndex", "1".toByteArray()))
        headers.add(RecordHeader("tableHeaderCount", "1".toByteArray()))
        headers.add(RecordHeader("tableHeaderIndex", "1".toByteArray()))
        headers.add(RecordHeader("tableIdentifierIndex", "1".toByteArray()))


Jonas Waeber's avatar
Jonas Waeber committed
87
        val testDriver = TopologyTestDriver(KafkaTopology(settings).prepare().build(), settings.kafkaStreamsSettings)
Jonas Waeber's avatar
Jonas Waeber committed
88
89
90
        val factory = ConsumerRecordFactory(
            StringSerializer(), StringSerializer()
        )
Jonas Waeber's avatar
Jonas Waeber committed
91
92
93
94
95
96
97
98
99
100


        testDriver.pipeInput(
            factory.create(
                "import-process-config",
                "test-record-set-id#mapping",
                readFile("kafkaTests/${params.count}/config/mapping.yml")
            )
        )

Jonas Waeber's avatar
Jonas Waeber committed
101
102
        testDriver.pipeInput(
            factory.create(
Jonas Waeber's avatar
Jonas Waeber committed
103
                settings.inputTopic, params.key, readFile("kafkaTests/${params.count}/input.json"), headers
Jonas Waeber's avatar
Jonas Waeber committed
104
105
106
            )
        )

Jonas Waeber's avatar
Jonas Waeber committed
107
108

        val record = testDriver.readOutput(
Jonas Waeber's avatar
Jonas Waeber committed
109
110
111
112
113
            settings.outputTopic,
            StringDeserializer(),
            StringDeserializer()
        )

Jonas Waeber's avatar
Jonas Waeber committed
114
115
116
117
118
119
120
121
122
123
        val model = ModelFactory.createDefaultModel()
        NS.prefixMapping.map {
            model.setNsPrefix(it.key, it.value)
        }
        RDFDataMgr.read(model, record.value().byteInputStream(), Lang.NTRIPLES)
        RDFDataMgr.write(
            FileOutputStream("$resourcePath/kafkaTests/${params.count}/turtle-output.ttl"),
            model,
            RDFFormat.TURTLE_PRETTY
        )
Jonas Waeber's avatar
Jonas Waeber committed
124

125
        val key = record.key()
Jonas Waeber's avatar
Jonas Waeber committed
126
        val sortedResult = sort(record.value().lines()).trim()
Jonas Waeber's avatar
Jonas Waeber committed
127

Jonas Waeber's avatar
Jonas Waeber committed
128
129
130
131
132
        val reportRecord = testDriver.readOutput(
            service.settings.processReportTopic,
            StringDeserializer(),
            StringDeserializer()
        )
133
134
135
136
137
138
139
140
141
142
143
144
145
146
        val reportString = reportRecord.value()

        assertAll("",
            { assertThat(key).isEqualTo(params.expectedKey) },
            {
                assertThat(sortedResult)
                    .isEqualTo(sort(output).trim())
            },
            {
                assertThat(Klaxon().parse<Report>(reportString))
                    .isEqualTo(params.report)
            }

        )
Jonas Waeber's avatar
Jonas Waeber committed
147
    }
Jonas Waeber's avatar
Jonas Waeber committed
148

Jonas Waeber's avatar
Jonas Waeber committed
149
    private fun kafkaTests() = Stream.of(
150
151
152
        /* IntegrationTestParams(
             1,
             "MEI_49884",
Jonas Waeber's avatar
Jonas Waeber committed
153
             "https://memobase.ch/record/test-record-set-id-MEI_49884",
154
             Report(
Jonas Waeber's avatar
Jonas Waeber committed
155
                 "https://memobase.ch/record/test-record-set-id-MEI_49884",
156
                 "SUCCESS",
Jonas Waeber's avatar
Jonas Waeber committed
157
                 "Successfully mapped record with id https://memobase.ch/record/test-record-set-id-MEI_49884."
158
159
160
             )

         ),
Jonas Waeber's avatar
Jonas Waeber committed
161
        IntegrationTestParams(
Jonas Waeber's avatar
Jonas Waeber committed
162
163
            2,
            "Sig Han 1293",
Jonas Waeber's avatar
Jonas Waeber committed
164
            "https://memobase.ch/record/test-record-set-id-Sig_Han_1293",
Jonas Waeber's avatar
Jonas Waeber committed
165
            Report(
Jonas Waeber's avatar
Jonas Waeber committed
166
                "https://memobase.ch/record/test-record-set-id-Sig_Han_1293",
Jonas Waeber's avatar
Jonas Waeber committed
167
                "SUCCESS",
Jonas Waeber's avatar
Jonas Waeber committed
168
                "Successfully mapped record with id https://memobase.ch/record/test-record-set-id-Sig_Han_1293."
169
            )
170
        ),
Jonas Waeber's avatar
Jonas Waeber committed
171
        IntegrationTestParams(
172
173
            3,
            "Sig Han 1293",
174
            "ttps://memobase.ch/record/TEST_RECORD_SET-Sig_Han_1293",
Jonas Waeber's avatar
Jonas Waeber committed
175
            Report(
Jonas Waeber's avatar
Jonas Waeber committed
176
                "https://memobase.ch/record/test-record-set-id-Sig_Han_1293",
Jonas Waeber's avatar
Jonas Waeber committed
177
                "SUCCESS",
Jonas Waeber's avatar
Jonas Waeber committed
178
                "Successfully mapped record with id https://memobase.ch/record/test-record-set-id-Sig_Han_1293."
179
            )
180
        ),
Jonas Waeber's avatar
Jonas Waeber committed
181
        IntegrationTestParams(
182
            4,
183
            "InputKey",
Jonas Waeber's avatar
Jonas Waeber committed
184
           "https://memobase.ch/record/test-record-set-id-sigantur-example",
Jonas Waeber's avatar
Jonas Waeber committed
185
            Report(
Jonas Waeber's avatar
Jonas Waeber committed
186
                "https://memobase.ch/record/test-record-set-id-sigantur-example",
Jonas Waeber's avatar
Jonas Waeber committed
187
                "SUCCESS",
Jonas Waeber's avatar
Jonas Waeber committed
188
                "Successfully mapped record with id https://memobase.ch/record/test-record-set-id-sigantur-example."
189
            )
190
        ),*/
Jonas Waeber's avatar
Jonas Waeber committed
191
        IntegrationTestParams(
192
193
            5,
            "ID_1",
Jonas Waeber's avatar
Jonas Waeber committed
194
            "https://memobase.ch/record/test-record-set-id-ID_1",
Jonas Waeber's avatar
Jonas Waeber committed
195
            Report(
Jonas Waeber's avatar
Jonas Waeber committed
196
                "https://memobase.ch/record/test-record-set-id-ID_1",
Jonas Waeber's avatar
Jonas Waeber committed
197
                "SUCCESS",
Jonas Waeber's avatar
Jonas Waeber committed
198
                "",
Jonas Waeber's avatar
Jonas Waeber committed
199
                Service.step
200
            )
201
        )/*,
Jonas Waeber's avatar
Jonas Waeber committed
202
        IntegrationTestParams(
Jonas Waeber's avatar
Jonas Waeber committed
203
204
            6,
            "ID_1",
Jonas Waeber's avatar
Jonas Waeber committed
205
           "https://memobase.ch/record/test-record-set-id-ID_1",
Jonas Waeber's avatar
Jonas Waeber committed
206
            Report(
Jonas Waeber's avatar
Jonas Waeber committed
207
                "https://memobase.ch/record/test-record-set-id-ID_1",
Jonas Waeber's avatar
Jonas Waeber committed
208
                "SUCCESS",
Jonas Waeber's avatar
Jonas Waeber committed
209
                "Successfully mapped record with id https://memobase.ch/record/test-record-set-id-ID_1."
Jonas Waeber's avatar
Jonas Waeber committed
210
            )
Jonas Waeber's avatar
Jonas Waeber committed
211
        ),
Jonas Waeber's avatar
Jonas Waeber committed
212
        IntegrationTestParams(
Jonas Waeber's avatar
Jonas Waeber committed
213
214
            7,
            "ID_1",
Jonas Waeber's avatar
Jonas Waeber committed
215
           "https://memobase.ch/record/test-record-set-id-ID_1",
Jonas Waeber's avatar
Jonas Waeber committed
216
            Report(
Jonas Waeber's avatar
Jonas Waeber committed
217
                "https://memobase.ch/record/test-record-set-id-ID_1",
Jonas Waeber's avatar
Jonas Waeber committed
218
                "SUCCESS",
Jonas Waeber's avatar
Jonas Waeber committed
219
                "Successfully mapped record with id https://memobase.ch/record/test-record-set-id-ID_1."
Jonas Waeber's avatar
Jonas Waeber committed
220
            )
221
        ),
Jonas Waeber's avatar
Jonas Waeber committed
222
        IntegrationTestParams(
223
224
            8,
            "ID_1",
Jonas Waeber's avatar
Jonas Waeber committed
225
           "https://memobase.ch/record/test-record-set-id-ID_1",
Jonas Waeber's avatar
Jonas Waeber committed
226
            Report(
Jonas Waeber's avatar
Jonas Waeber committed
227
                "https://memobase.ch/record/test-record-set-id-ID_1",
Jonas Waeber's avatar
Jonas Waeber committed
228
                "SUCCESS",
Jonas Waeber's avatar
Jonas Waeber committed
229
                "Successfully mapped record with id https://memobase.ch/record/test-record-set-id-ID_1."
230
            )
Jonas Waeber's avatar
Jonas Waeber committed
231
        ),
Jonas Waeber's avatar
Jonas Waeber committed
232
        IntegrationTestParams(
Jonas Waeber's avatar
Jonas Waeber committed
233
234
            9,
            "ID_1",
Jonas Waeber's avatar
Jonas Waeber committed
235
           "https://memobase.ch/record/test-record-set-id-ID_1",
Jonas Waeber's avatar
Jonas Waeber committed
236
            Report(
Jonas Waeber's avatar
Jonas Waeber committed
237
                "https://memobase.ch/record/test-record-set-id-ID_1",
Jonas Waeber's avatar
Jonas Waeber committed
238
                "SUCCESS",
Jonas Waeber's avatar
Jonas Waeber committed
239
                "Successfully mapped record with id https://memobase.ch/record/test-record-set-id-ID_1."
Jonas Waeber's avatar
Jonas Waeber committed
240
            )
241
        )*/
Jonas Waeber's avatar
Jonas Waeber committed
242
    )
Jonas Waeber's avatar
Jonas Waeber committed
243
}