KafkaTopology.kt 2.38 KB
Newer Older
Jonas Waeber's avatar
Jonas Waeber committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
/*
 * Table Data Import Service
 * Copyright (C) 2020 Memoriav
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */

package org.memobase

import com.beust.klaxon.JsonArray
import com.beust.klaxon.JsonObject
import com.beust.klaxon.Klaxon
import java.io.StringReader
import org.apache.kafka.streams.StreamsBuilder
import org.apache.kafka.streams.Topology
import org.apache.logging.log4j.LogManager
import org.memobase.settings.SettingsLoader

class KafkaTopology(private val settings: SettingsLoader) {
    private val log = LogManager.getLogger("StreamsProcessing")

    private val searchDocTransform = SearchDocTransform(settings.appSettings.getProperty("mapping"))
    fun build(): Topology {
        val builder = StreamsBuilder()
        val stream = builder.stream<String, String>(settings.inputTopic)
        stream
            .flatMapValues { value -> parseJson(value) }
            .mapValues { value -> unpackJson(value) }
            .mapValues { value -> transformJson(value) }
            .mapValues { value -> value.toJsonString() }
            .to(settings.outputTopic)

        return builder.build()
    }

    private fun parseJson(data: String): List<JsonObject> {
        val result = Klaxon().parseJsonObject(StringReader(data))
        return if (result == null) emptyList()
        else listOf(result)
    }

    private fun unpackJson(input: JsonObject): Map<String, JsonObject> {
        val graph = input["@graph"] as JsonArray<JsonObject>
        return graph.map {
            if (it["@type"] == "rico:Record") {
                Pair("record", it)
            } else {
                Pair(it["@id"] as String, it)
            }
        }.toMap()
    }

    private fun transformJson(input: Map<String, JsonObject>): JsonObject {
        return searchDocTransform.transform(input)
    }
}