Add docker compose and some scripts to run the whole application locally with...

Add docker compose and some scripts to run the whole application locally with kafka and mongo in docker compose

Makefile

+# these will speed up builds, for docker-compose >= 1.25
+export COMPOSE_DOCKER_CLI_BUILD=1
+export DOCKER_BUILDKIT=1
+
+all: down build up test
+
+build:
+	docker compose build
+
+buildnocache:
+	docker compose build --no-cache
+
+up:
+	docker compose up -d
+
+down:
+	docker compose down --remove-orphans
+
+test:
+	sbt run test

README.md

 # Browse Index Values
 
 This code creates the solr records for the swisscollections "Index Search"
+
+## Test locally with kafka cluster
+
+make up (will start a kafka cluster via docker-compose)
+
+Create the topics and the test data
+```
+./scripts/create-test-data.sh
+```
+
+Set the following environment variables
+
+```
+APPLICATION_ID=browse-values
+KAFKA_BOOTSTRAP_SERVERS=localhost:29092
+TOPIC_IN=swisscollections-slsp-deduplicated
+TOPIC_OUT=swisscollections-browse-values
+LEECH_XSL_TEMPLATE=leech.xslt
+TRANSFORMER_IMPL=net.sf.saxon.TransformerFactoryImpl
+MONGO_COLLECTION=sourceDNBGND1
+MONGO_DB=nativeSources
+MONGO_URI=mongodb://localhost:27013
+```
+
+
+Run the program (click the green arrow in App.scala in IntelliJ) or
+```
+sbt run
+```
+
+Check what is written in swisscollections-browse-values (use the kafka plugin in intellij to read from topics)
+
+
+See readme in slsp-deduplication for more background
+

data/records.txt

+(EXLNZ-41SLSP_NETWORK)991094525059705501|<record><metadata><record><leader>01795nam a2200433 c 4500</leader><controlfield tag="001">991094525059705501</controlfield><controlfield tag="005">20230805162721.0</controlfield><controlfield tag="007">cr#|||||||||||</controlfield><controlfield tag="008">100614s1791 sz00| | ger</controlfield><datafield tag="019" ind1=" " ind2=" "><subfield code="a">Ergänzt nach Vorlage (Ex. d. UB Basel; im VD18 nicht aufgeführt)</subfield><subfield code="5">IDSBB/07.06.2022/A100/HPF</subfield></datafield><datafield tag="019" ind1=" " ind2=" "><subfield code="a">Unikatsaufnahme, gesperrt für das Anhängen weiterer Signaturen</subfield><subfield code="5">IDSBB/13.08.2020/bv,cpi</subfield></datafield><datafield tag="024" ind1="7" ind2=" "><subfield code="a">10.3931/e-rara-92019</subfield><subfield code="2">doi</subfield></datafield><datafield tag="035" ind1=" " ind2=" "><subfield code="a">(swissbib)063625695-41slsp_network</subfield></datafield><datafield tag="035" ind1=" " ind2=" "><subfield code="a">063625695</subfield><subfield code="9">ExL</subfield></datafield><datafield tag="035" ind1=" " ind2=" "><subfield code="a">(IDSBB)005245183DSV01</subfield></datafield><datafield tag="035" ind1=" " ind2=" "><subfield code="a">(EXLNZ-41SLSP_NETWORK)991094525059705501</subfield></datafield><datafield tag="035" ind1=" " ind2=" "><subfield code="a">(41SLSP_UBS)9952451830105504</subfield><subfield code="9">(41SLSP_UBS)9952451830105504</subfield></datafield><datafield tag="040" ind1=" " ind2=" "><subfield code="a">SzZuIDS BS/BE A110</subfield><subfield code="e">rda</subfield><subfield code="d">CH-ZuSLS</subfield></datafield><datafield tag="100" ind1="1" ind2=" "><subfield code="a">La Roche, Andreas</subfield><subfield code="d">1757-1817</subfield><subfield code="0">(DE-588)106464399X</subfield><subfield code="4">aut</subfield></datafield><datafield tag="245" ind1="1" ind2="0"><subfield code="a">Leichenrede über Psalm CXXVI. 5. 6. bey der Beerdigung der ... Margaretha Raillard</subfield><subfield code="b">Freytags den 14. Jenner 1791 im Münster gehalten</subfield><subfield code="c">von Andreas La Roche Diakonus bey St. Peter</subfield></datafield><datafield tag="264" ind1=" " ind2="1"><subfield code="a">Basel</subfield><subfield code="b">gedruckt bey Wilhelm Haas, dem Sohne</subfield><subfield code="c">[1791?]</subfield></datafield><datafield tag="300" ind1=" " ind2=" "><subfield code="a">20 Seiten</subfield><subfield code="c">8°</subfield></datafield><datafield tag="336" ind1=" " ind2=" "><subfield code="b">txt</subfield><subfield code="2">rdacontent</subfield></datafield><datafield tag="337" ind1=" " ind2=" "><subfield code="b">n</subfield><subfield code="2">rdamedia</subfield></datafield><datafield tag="337" ind1=" " ind2=" "><subfield code="b">c</subfield><subfield code="2">rdamedia</subfield></datafield><datafield tag="338" ind1=" " ind2=" "><subfield code="b">nc</subfield><subfield code="2">rdacarrier</subfield></datafield><datafield tag="338" ind1=" " ind2=" "><subfield code="b">cr</subfield><subfield code="2">rdacarrier</subfield></datafield><datafield tag="500" ind1=" " ind2=" "><subfield code="a">Gelegenheitsschrift: Tod</subfield></datafield><datafield tag="500" ind1=" " ind2=" "><subfield code="a">Lebensdaten: 16.11.1750-12.01.1791</subfield></datafield><datafield tag="500" ind1=" " ind2=" "><subfield code="a">Bestattungsdatum: 14.01.1791</subfield></datafield><datafield tag="655" ind1=" " ind2="7"><subfield code="a">Nachruf</subfield><subfield code="2">gnd-content</subfield></datafield><datafield tag="700" ind1="1" ind2=" "><subfield code="a">Raillard, Margaretha</subfield><subfield code="d">1750-1791</subfield><subfield code="e">Gefeierter</subfield><subfield code="4">hnr</subfield><subfield code="0">(DE-588)1295776367</subfield></datafield><datafield tag="700" ind1="1" ind2=" "><subfield code="a">Haas, Wilhelm</subfield><subfield code="d">1766-1838</subfield><subfield code="e">Drucker</subfield><subfield code="4">prt</subfield><subfield code="0">(DE-588)11955593X</subfield></datafield><datafield tag="751" ind1=" " ind2=" "><subfield code="a">Basel</subfield><subfield code="0">(DE-588)4004617-5</subfield></datafield><datafield tag="852" ind1="4" ind2=" "><subfield code="b">A348</subfield><subfield code="c">FH</subfield><subfield code="j">STA LA 1791 01 12</subfield><subfield code="8">22196573380005504</subfield><subfield code="9">(41SLSP_UBS)9952451830105504</subfield></datafield><datafield tag="852" ind1="4" ind2=" "><subfield code="b">A100</subfield><subfield code="c">MAG</subfield><subfield code="j">UBH</subfield><subfield code="x">Holding für Migration per Programm erstellt</subfield><subfield code="8">22196573410005504</subfield><subfield code="9">(41SLSP_UBS)9952451830105504</subfield></datafield><datafield tag="852" ind1="4" ind2=" "><subfield code="b">A116</subfield><subfield code="c">116B2</subfield><subfield code="j">-</subfield><subfield code="8">22401587030005504</subfield><subfield code="9">(41SLSP_UBS)9952451830105504</subfield></datafield><datafield tag="856" ind1="4" ind2="1"><subfield code="u">http://dx.doi.org/10.3931/e-rara-92019</subfield><subfield code="z">Online via e-rara</subfield></datafield><datafield tag="866" ind1=" " ind2=" "><subfield code="a">-</subfield><subfield code="8">22196573410005504</subfield></datafield><datafield tag="900" ind1=" " ind2=" "><subfield code="a">IDSIxunikat</subfield></datafield><datafield tag="900" ind1=" " ind2=" "><subfield code="a">IDSIxkasualia</subfield></datafield><datafield tag="900" ind1=" " ind2=" "><subfield code="a">NOMERGEALEX</subfield></datafield><datafield tag="949" ind1=" " ind2=" "><subfield code="3">22196573410005504</subfield><subfield code="o">BOOK</subfield><subfield code="2">DSVN7955637</subfield><subfield code="c">MAG</subfield><subfield code="h">UBH KiAr G X 84:2:7</subfield><subfield code="1">23196573390005504</subfield><subfield code="x">R</subfield><subfield code="p">11</subfield><subfield code="j">UBH</subfield><subfield code="b">A100</subfield><subfield code="9">(41SLSP_UBS)9952451830105504</subfield></datafield><datafield tag="949" ind1=" " ind2=" "><subfield code="3">22196573410005504</subfield><subfield code="o">BOOK</subfield><subfield code="2">DSVN7665047</subfield><subfield code="c">MAG</subfield><subfield code="h">UBH KiAr G X 71:12</subfield><subfield code="1">23196573400005504</subfield><subfield code="x">R</subfield><subfield code="p">11</subfield><subfield code="j">UBH</subfield><subfield code="b">A100</subfield><subfield code="9">(41SLSP_UBS)9952451830105504</subfield></datafield><datafield tag="949" ind1=" " ind2=" "><subfield code="3">22196573380005504</subfield><subfield code="o">BOOK</subfield><subfield code="x">REKATA1101006</subfield><subfield code="2">DSVN4804400</subfield><subfield code="c">FH</subfield><subfield code="1">23196573370005504</subfield><subfield code="x">R</subfield><subfield code="p">68</subfield><subfield code="j">STA LA 1791 01 12</subfield><subfield code="b">A348</subfield><subfield code="9">(41SLSP_UBS)9952451830105504</subfield></datafield><datafield tag="949" ind1=" " ind2=" "><subfield code="3">22401587030005504</subfield><subfield code="o">BOOK</subfield><subfield code="2">UBS-S-794459</subfield><subfield code="c">116B2</subfield><subfield code="h">-</subfield><subfield code="1">23401587020005504</subfield><subfield code="p">67</subfield><subfield code="j">-</subfield><subfield code="b">A116</subfield><subfield code="9">(41SLSP_UBS)9952451830105504</subfield></datafield><datafield tag="960" ind1=" " ind2=" "><subfield code="a">Raillard, Margaretha (1750-1791)</subfield><subfield code="2">ids-AW</subfield><subfield code="9">local</subfield><subfield code="9">(41SLSP_UBS)9952451830105504</subfield></datafield><datafield tag="986" ind1=" " ind2=" "><subfield code="a">(41SLSP_UBS)9952451830105504</subfield><subfield code="9">(41SLSP_UBS)9952451830105504</subfield></datafield><datafield tag="990" ind1=" " ind2=" "><subfield code="c">BBKSKAUT</subfield><subfield code="9">local</subfield><subfield code="9">(41SLSP_UBS)9952451830105504</subfield></datafield><datafield tag="990" ind1=" " ind2=" "><subfield code="f">erarabs</subfield><subfield code="9">local</subfield><subfield code="9">(41SLSP_UBS)9952451830105504</subfield></datafield></record></metadata></record>

docker-compose.yml

+---
+services:
+  broker:
+    # from https://developer.confluent.io/confluent-tutorials/kafka-on-docker/
+    image: apache/kafka:latest
+    hostname: broker
+    ports:
+      - 9092:9092
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT,CONTROLLER:PLAINTEXT
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+      KAFKA_PROCESS_ROLES: broker,controller
+      KAFKA_NODE_ID: 1
+      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@broker:29093
+      KAFKA_LISTENERS: PLAINTEXT://broker:29092,CONTROLLER://broker:29093,PLAINTEXT_HOST://0.0.0.0:9092
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
+      KAFKA_LOG_DIRS: /tmp/kraft-combined-logs
+      CLUSTER_ID: MkU3OEVBNTcwNTJENDM2Qk
+    working_dir: /data
+    volumes:
+      - ./data:/data
+
+  mongo:
+    image: mongo
+    ports:
+      - "27013:27017"
+    environment:
+      MONGO_INITDB_DATABASE: nativeSources

scripts/create-test-data.sh

+# create topics and load data
+
+# delete topics
+docker compose exec broker bash -c "
+/opt/kafka/bin/kafka-topics.sh \
+  --bootstrap-server broker:29092 \
+  --topic swisscollections-slsp-deduplicated \
+  --delete"
+
+docker compose exec broker bash -c "
+/opt/kafka/bin/kafka-topics.sh \
+  --bootstrap-server broker:29092 \
+  --topic swisscollections-slsp-deduplicated \
+  --delete"
+
+# create topic
+docker compose exec broker bash -c "
+/opt/kafka/bin/kafka-topics.sh \
+  --bootstrap-server broker:29092 \
+  --topic swisscollections-slsp-deduplicated \
+  --replication-factor 1 \
+  --partitions 4 \
+  --create"
+
+# create topic
+docker compose exec broker bash -c "
+/opt/kafka/bin/kafka-topics.sh \
+  --bootstrap-server broker:29092 \
+  --topic swisscollections-browse-values \
+  --replication-factor 1 \
+  --partitions 4 \
+  --create"
+
+# produce data
+docker compose exec broker bash -c "
+  /opt/kafka/bin/kafka-console-producer.sh \
+  --bootstrap-server broker:29092 \
+  --topic swisscollections-slsp-deduplicated \
+  --property 'parse.key=true' \
+  --property 'key.separator=|' < records.txt"