Merge branch 'update-deployment' into 'master'

Update helm-charts and values

See merge request !13

build.gradle

@@ -20,10 +20,9 @@ sourceCompatibility = 1.8
 targetCompatibility = 1.8
 
 repositories {
-    jcenter()
     mavenCentral()
     maven {
-        url  "https://dl.bintray.com/memoriav/memobase"
+        url  "https://gitlab.switch.ch/api/v4/projects/1324/packages/maven"
     }
 }
 
@@ -36,7 +35,7 @@ dependencies {
     def excludeSlf4J = { exclude group: 'org.slf4j' }
 
     // JSON Parser
-    implementation 'com.beust:klaxon:5.2'
+    implementation 'com.beust:klaxon:5.5'
 
     implementation "org.apache.logging.log4j:log4j-api:${log4jV}"
     implementation "org.apache.logging.log4j:log4j-core:${log4jV}"
@@ -46,8 +45,8 @@ dependencies {
 
     implementation 'org.apache.jena:apache-jena:3.14.0', excludeSlf4J
 
-    implementation 'org.memobase:memobase-service-utilities:2.0.2', excludeSlf4J
-    implementation 'ch.memobase:fedora-client:0.7.0', excludeSlf4J
+    implementation 'org.memobase:memobase-service-utilities:3.0.1', excludeSlf4J
+    implementation 'ch.memobase:fedora-client:0.8.0', excludeSlf4J
 
     // KOTLIN IMPORTS
     implementation 'org.jetbrains.kotlin:kotlin-stdlib-jdk8'

helm-charts/Chart.yaml

 apiVersion: v2
 name: fedora-ingest-service
-description: A helm chart for the fedora-ingest-service service.
+description: A helm chart for the fedora-ingest-service service
 type: application
 version: 0.0.0
 appVersion: 0.0.0
 
 maintainers:
   - name: Thomas Bernhart
-    email: t.bernhart@docuteam.ch
\ No newline at end of file
+    email: t.bernhart@docuteam.ch

helm-charts/helm-values/ip-fedora-ingest.yaml → helm-charts/helm-values/di-fedora-indexer-prod.yml

-deploymentName: ip-ingest-service
-clientId: fedora-ingest-service-app
+deploymentName: di-fedora-indexer-prod
+
+k8sName: fedora-indexer
+k8sNamespace: memobase
+k8sGroupId: di
+k8sGroupName: documents-import
+k8sEnvironment: prod
+k8sReplicas: 1
+k8sRequestsCpu: "0.2"
+k8sRequestsMemory: "0.2Gi"
+k8sLimitsCpu: "1"
+k8sLimitsMemory: "1Gi"
 
 kafkaConfigs: prod-kafka-bootstrap-servers
 sftpConfigs: internal-sftp-config
 fedoraConfigs: fedora-admin-credentials
 isSimple: false
 
-inputTopicName: import-process-ingest
+inputTopicName: mb-di-ingest-prod
 outputTopicName: void_topic
-reportingTopicName: import-process-reporting
+reportingTopicName: mb-di-reporting-prod
+
+clientId: prod-di-fedora-indexer
\ No newline at end of file

helm-charts/helm-values/dp-fedora-ingest.yml → helm-charts/helm-values/gi-fedora-indexer-prod.yml

-deploymentName: dp-sync-ingest
-clientId: fedora-drupal-sync-ingest-app
+deploymentName: gi-fedora-indexer-prod
+
+k8sName: fedora-indexer
+k8sNamespace: memobase
+k8sGroupId: gi
+k8sGroupName: group-import
+k8sEnvironment: prod
+k8sReplicas: 1
+k8sRequestsCpu: "0.2"
+k8sRequestsMemory: "0.2Gi"
+k8sLimitsCpu: "1"
+k8sLimitsMemory: "1Gi"
 
 kafkaConfigs: prod-kafka-bootstrap-servers
 sftpConfigs: internal-sftp-config
 fedoraConfigs: fedora-admin-credentials
 isSimple: true
 
-inputTopicName: drupal-sync-output
+inputTopicName: mb-gi-rdf-objects-prod
 outputTopicName: void_topic
-reportingTopicName: import-process-reporting
+reportingTopicName: mb-di-reporting-prod
+
+clientId: prod-gi-fedora-indexer

helm-charts/templates/app-config.yaml

 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: "{{ .Values.deploymentName }}-config"
-  namespace: memobase
-data:
-  CLIENT_ID: {{ .Values.clientId }}
-  IS_SIMPLE: "{{ .Values.isSimple }}"
-  TOPIC_IN: {{ .Values.inputTopicName }}
-  TOPIC_OUT: {{ .Values.outputTopicName }}
-  TOPIC_REPORTING: {{ .Values.reportingTopicName }}
+  name: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-config"
+  namespace: "{{ .Values.k8sNamespace }}"
+  data:
+    CLIENT_ID: {{ .Values.clientId }}
+    IS_SIMPLE: "{{ .Values.isSimple }}"
+    TOPIC_IN: {{ .Values.inputTopicName }}
+    TOPIC_OUT: {{ .Values.outputTopicName }}
+    TOPIC_REPORTING: {{ .Values.reportingTopicName }}

helm-charts/templates/deployment.yaml

 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: {{ .Values.deploymentName }}-deployment
+  name: {{ .Values.deploymentName }}
   namespace: memobase
   labels:
-    jobType: "import-process-deployment"
+    app: {{ .Values.deploymentName }}-app
 spec:
   selector:
     matchLabels:
-      app: {{ .Values.deploymentName }}
+      app: {{ .Values.deploymentName }}-app
   replicas: 1
   template:
     metadata:
       labels:
-        app: {{ .Values.deploymentName }}
-        tier: import-process
+        app: {{ .Values.deploymentName }}-app
+        tier: import-process #FIXME: web?
     spec:
-      restartPolicy: Always
       containers:
-      - name: "{{ .Values.deploymentName }}-container"
-        image: "{{ .Values.registry }}/{{ .Values.image }}:{{ .Values.tag }}"
-        imagePullPolicy: Always
-        resources:
-          requests:
-            cpu: "0.5"
-            memory: "128Mi"
-          limits:
-            cpu: "1"
-            memory: "1Gi"
-        envFrom:
-        - secretRef:
-            name: "{{ .Values.sftpConfigs }}"
-        - secretRef:
-            name: "{{ .Values.fedoraConfigs }}"
-        - configMapRef:
-            name: "{{ .Values.kafkaConfigs }}"
-        - configMapRef:
-            name: "{{ .Values.deploymentName }}-config"
+        - name: "{{ .Values.deploymentName }}-container"
+          image: "{{ .Values.registry }}/{{ .Values.image }}:{{ .Values.tag }}"
+          imagePullPolicy: Always
+          resources:
+            requests:
+              cpu: "0.5"
+              memory: "128Mi"
+            limits:
+              cpu: "1"
+              memory: "1Gi"
+          envFrom:
+          - secretRef:
+              name: "{{ .Values.sftpConfigs }}"
+          - secretRef:
+              name: "{{ .Values.fedoraConfigs }}"
+          - configMapRef:
+              name: "{{ .Values.kafkaConfigs }}"
+          - configMapRef:
+              name: "{{ .Values.deploymentName }}-config"
+      restartPolicy: Always

helm-charts/values.yaml

@@ -3,14 +3,15 @@ registry: "cr.gitlab.switch.ch"
 image: "memoriav/memobase-2020/services/import-process/fedora-ingest-service"
 tag: "latest"
 
-deploymentName: fedora-ingest-service
-clientId: fedora-ingest-service-app
+deploymentName: placeholder
 
-kafkaConfigs: prod-kafka-bootstrap-servers
-sftpConfigs: internal-sftp-config
-fedoraConfigs: fedora-admin-credentials
-isSimple: true
+kafkaConfigs: placeholder
+sftpConfigs: placeholder
+fedoraConfigs: placeholder
+isSimple: placeholder
 
-inputTopicName: import-process-normalization
-outputTopicName: void_topic
-reportingTopicName: import-process-reporting
+inputTopicName: placeholder
+outputTopicName: placeholder
+reportingTopicName: placeholder
+
+clientId: placeholder
\ No newline at end of file

src/main/kotlin/Service.kt

@@ -39,8 +39,7 @@ class Service(fileName: String = "app.yml") : Closeable {
     companion object {
         const val FEDORA_PROPERTIES_PREFIX = "fedora"
         const val CONSUMER_MAX_POLL_RECORDS = "10"
-        const val CONSUMER_MAX_INTERVAL_MS =  "600000" // 600000ms = 10min
-
+        const val CONSUMER_MAX_INTERVAL_MS = "600000" // 600000ms = 10min
         const val BINARY_FILE_URI_PATH = "binary"
         const val SFTP_PREFIX = "sftp:"