update kafka version; provide jobs for stage & test

k8s-manifests/test/job.yml → k8s-manifests/test/job-prod.yml

 apiVersion: batch/v1
 kind: Job
 metadata:
-  name: import-process-delete
+  name: dd-marker-prod
   namespace: memobase
   labels:
-    app: import-process-delete-app
+    app: dd-marker-prod-app
 spec:
   template:
     metadata:
       labels:
-        app: import-process-delete-app
-        tier: web
+        app: dd-marker-prod-app
+        group: documents-delete
+        environment: prod
     spec:
       containers:
-        - name: import-process-delete-container
+        - name: dd-marker-prod-container
           args: [ # Customise to match your needs
               "--record-set-filter", "<id>",
               "--record-filter", "<id>",
@@ -30,11 +31,11 @@ spec:
               - name: "prod-kafka-bootstrap-servers"
           env:
             - name: CLIENT_ID
-              value: import-process-delete
+              value: dd-marker-prod
             - name: TOPIC_IN
-              value: import-process-transactions-records
+              value: mb-di-transactions-records-prod
             - name: TOPIC_OUT
-              value: delete-process-deletes
+              value: mb-dd-deletes-prod
             - name: POLL_TIMEOUT
               value: "20000"
       restartPolicy: Never

k8s-manifests/test/job-stage.yml

+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: dd-marker-stage
+  namespace: memobase
+  labels:
+    app: dd-marker-stage-app
+spec:
+  template:
+    metadata:
+      labels:
+        app: dd-marker-stage-app
+        group: documents-delete
+        environment: stage
+    spec:
+      containers:
+        - name: dd-marker-stage-container
+          args: [ # Customise to match your needs
+              "--record-set-filter", "<id>",
+              "--record-filter", "<id>",
+              "--institution-filter", "<id>",
+              "--session-filter", "<id>",
+              "--created-after", "<datetime>",
+              "--created-before", "<datetime>"
+              "<your-session-id>"
+          ]
+          image: cr.gitlab.switch.ch/memoriav/memobase-2020/services/deletion-components/import-process-delete:latest
+          imagePullPolicy: Always
+          envFrom:
+            - configMapRef:
+              - name: "prod-kafka-bootstrap-servers"
+          env:
+            - name: CLIENT_ID
+              value: dd-marker-stage
+            - name: TOPIC_IN
+              value: mb-di-transactions-records-stage
+            - name: TOPIC_OUT
+              value: mb-dd-deletes-stage
+            - name: POLL_TIMEOUT
+              value: "20000"
+      restartPolicy: Never
+  backoffLimit: 1
+

k8s-manifests/test/job-test.yml

+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: dd-marker-test
+  namespace: memobase
+  labels:
+    app: dd-marker-test-app
+spec:
+  template:
+    metadata:
+      labels:
+        app: dd-marker-test-app
+        group: documents-delete
+        environment: test
+    spec:
+      containers:
+        - name: dd-marker-test-container
+          args: [ # Customise to match your needs
+              "--record-set-filter", "<id>",
+              "--record-filter", "<id>",
+              "--institution-filter", "<id>",
+              "--session-filter", "<id>",
+              "--created-after", "<datetime>",
+              "--created-before", "<datetime>"
+              "<your-session-id>"
+          ]
+          image: cr.gitlab.switch.ch/memoriav/memobase-2020/services/deletion-components/import-process-delete:latest
+          imagePullPolicy: Always
+          envFrom:
+            - configMapRef:
+              - name: "test-kafka-bootstrap-servers"
+          env:
+            - name: CLIENT_ID
+              value: dd-marker-test
+            - name: TOPIC_IN
+              value: mb-di-transactions-records-prod
+            - name: TOPIC_OUT
+              value: mb-dd-deletes-prod
+            - name: POLL_TIMEOUT
+              value: "20000"
+      restartPolicy: Never
+  backoffLimit: 1
+

project/Dependencies.scala

@@ -19,7 +19,7 @@
 import sbt._
 
 object Dependencies {
-  lazy val kafkaV = "2.3.1"
+  lazy val kafkaV = "2.7.0"
   lazy val log4jV = "2.11.2"
   lazy val scalatestV = "3.1.2"