replace k8s manifest with helm chart

.gitignore

@@ -13,5 +13,6 @@ bin
 *.tmp
 gh/testdaten
 .idea
+.metals
 node_modules
-conf/development.conf
\ No newline at end of file
+conf/development.conf

.gitlab-ci.yml

 stages:
+  - test
   - test-build
   - publish
 
@@ -17,3 +18,5 @@ include:
   - project: 'memoriav/memobase-2020/utilities/ci-templates'
     ref: modular
     file: 'docker/default.yml'
+  - project: 'memoriav/memobase-2020/utilities/ci-templates'
+    file: 'helm-chart/helm-chart.yml'

app/modules/ElasticsearchClient.scala

@@ -49,7 +49,7 @@ class ElasticsearchClient @Inject()(
     Future.successful(client.getOrElse(Option.empty))
   })
 
-  override val index: String = getEnv("ELASTICSEARCH_INDEX")
+  override val index: String = getEnv("ELASTIC_INDEX")
 
   //val client: Option[RestHighLevelClient] = connect()
   override val client: Option[RestHighLevelClient] = connect()
@@ -117,14 +117,14 @@ class ElasticsearchClient @Inject()(
 
   private def connect(): Option[RestHighLevelClient] = {
     val hosts = new ArrayBuffer[HttpHost]
-    val configuredHosts = getEnv("ELASTICSEARCH_HOSTS").split(";")
-    val configuredPort = getEnv("ELASTICSEARCH_PORT").toIntOption.getOrElse(8080)
+    val configuredHosts = getEnv("ELASTIC_HOST").split(";")
+    val configuredPort = getEnv("ELASTIC_PORT").toIntOption.getOrElse(8080)
     configuredHosts.foreach(
       value => {
         hosts += new HttpHost(value, configuredPort)
       }
     )
-    val headers = Array(new BasicHeader("cluster.name", getEnv("ELASTICSEARCH_CLUSTER"))
+    val headers = Array(new BasicHeader("cluster.name", getEnv("ELASTIC_CLUSTERNAME"))
       .asInstanceOf[Header])
     Option(new RestHighLevelClient(RestClient.builder(hosts.toArray : _*).setDefaultHeaders(headers)))
   }

helm-charts/.helmignore

+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
\ No newline at end of file

helm-charts/Chart.yaml

+apiVersion: v2
+name: oai
+description: A helm chart for the OAI service
+type: application
+version: 0.0.0
+appVersion: 0.0.0
+
+
+
+maintainers:
+  - name: Sebastian Schüpbach
+    email: sebastian.schuepbach@unibas.ch

helm-charts/helm-values/api-oai-prod.yaml

+tag: "latest"
+
+k8sEnvironment: prod
+k8sReplicas: 1
+k8sRequestsCpu: placeholder # TODO: Set requests
+k8sRequestsMemory: placeholder # TODO: Set requests
+k8sLimitsCpu: placeholder # TODO: Set limits
+k8sLimitsMemory: placeholder # TODO: Set limits
+k8sHost: oai.memobase.k8s.unibas.ch
+k8sPort: 9000
+
+elasticHostConfigs: prod-elastic-configs
+elasticIndex: "oai-v9"
+elasticClustername: "test-memobase-search-cluster" # TODO: Is this still needed?
+
+responseListlength: "30"
+resumptionTokenTtl: "3"

helm-charts/helm-values/api-oai-stage.yaml

+tag: "latest"
+
+k8sEnvironment: stage
+k8sReplicas: 1
+k8sRequestsCpu: placeholder # TODO: Set requests
+k8sRequestsMemory: placeholder # TODO: Set requests
+k8sLimitsCpu: placeholder # TODO: Set limits
+k8sLimitsMemory: placeholder # TODO: Set limits
+k8sHost: oai-stage.memobase.k8s.unibas.ch
+k8sPort: 9000
+
+elasticHostConfigs: stage-elastic-configs
+elasticIndex: "oai-v9" # TODO: Set correct stage index name
+elasticClustername: "test-memobase-search-cluster" # TODO: Is this still needed?
+
+responseListlength: "30"
+resumptionTokenTtl: "3"

helm-charts/helm-values/api-oai-test.yaml

+tag: "latest"
+
+k8sEnvironment: test
+k8sReplicas: 1
+k8sRequestsCpu: placeholder # TODO: Set requests
+k8sRequestsMemory: placeholder # TODO: Set requests
+k8sLimitsCpu: placeholder # TODO: Set limits
+k8sLimitsMemory: placeholder # TODO: Set limits
+k8sHost: oai-test.memobase.k8s.unibas.ch
+k8sPort: 9000
+
+elasticHostConfigs: test-elastic-configs
+elasticIndex: "oai-v9" # TODO: Set correct test index name
+elasticClustername: "test-memobase-search-cluster" # TODO: Is this still needed?
+
+responseListlength: "30"
+resumptionTokenTtl: "3"

helm-charts/prod-oai.conf

+oaiconfigs: {
+  common: {
+    xsi_schemaLocation: "http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd"
+
+  }
+  sets: [
+    {
+        "spec": "memobase"
+        "name": "Association of all sets defined by Memobase for OAI export"
+        "includedsets": ["memobase:apf"]
+    }
+
+    {
+        "spec": "memobase:europeana"
+        "name": "Association of all sets defined by Memobase for OAI export to Europeana"
+        "includedsets": ["memobase:apf"]
+    }
+
+    {
+        #brauchen wir das auf Bestandesebene? - dann ein includedsets
+        "spec": "memobase:apf"
+        "name": "Association Films Plans-Fixes, all record sets"
+        "includedsets": ["memobase:apf-001"]
+        #"field": "institution"
+        #"value": "apf"
+    }
+
+    {
+        #brauchen wir das auf Bestandesebene? - dann ein includedsets
+        "spec": "memobase:apf-001"
+        "name": "Association Films Plans-Fixes, Fonds film et video Plans-Fixes"
+        "field": "recordset"
+        "value": "apf-001"
+    }
+
+
+  ]
+  metadataPrefix: [
+    {
+      "metadataPrefix": "edm"
+      "schema": "http://www.europeana.eu/schemas/edm/EDM.xsd"
+      "metadataNamespace": "https://pro.europeana.eu/page/edm-documentation"
+    }
+
+  ]
+  identify: {
+    repositoryName: "memobase OAI repositories"
+    baseURL: "https://oai.memobase.ch"
+    protocolVersion: "2.0"
+    adminEmail: "admin@memobase.ch"
+    earliestDatestamp: "2021-01-01T00:00:00.000Z"
+    deletedRecord: "transient"
+    granularity: "YYYY-MM-DDThh:mm:ss.sssZ"
+  }
+}

helm-charts/prod-secure.conf

+# Set up Play for HTTPS and locked down allowed hosts.
+# Nothing in here is required for REST, but it's a good default.
+play {
+  http {
+    cookies.strict = true
+
+    session.secure = true
+    session.httpOnly = true
+
+    flash.secure = true
+    flash.httpOnly = true
+
+    forwarded.trustedProxies = ["::1", "127.0.0.1"]
+  }
+
+  i18n {
+    langCookieSecure = true
+    langCookieHttpOnly = true
+  }
+
+  filters {
+    csrf {
+      cookie.secure = true
+    }
+
+
+    hosts {
+
+      # A list of valid hosts (e.g. "example.com") or suffixes of valid hosts (e.g. ".example.com")
+      # Note that ".example.com" will match example.com and any subdomain of example.com, with or without a trailing dot.
+      # "." matches all domains, and "" matches an empty or nonexistent host.
+      allowed = ["192.168.99.100", "localhost", ".k8s.unibas.ch", "127.0.0.1", ".memobase.ch"]
+
+      routeModifiers {
+        # If non empty, then requests will be checked if the route does not have this modifier. This is how we enable the
+        # anyhost modifier, but you may choose to use a different modifier (such as "api") if you plan to check the
+        # modifier in your code for other purposes.
+        whiteList = ["anyhost"]
+
+        # If non empty, then requests will be checked if the route contains this modifier
+        # The black list is used only if the white list is empty
+        blackList = []
+      }
+    }
+  }
+}
\ No newline at end of file

helm-charts/stage-oai.conf

+oaiconfigs: {
+  common: {
+    xsi_schemaLocation: "http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd"
+
+  }
+  sets: [
+    {
+        "spec": "memobase"
+        "name": "Association of all sets defined by Memobase for OAI export"
+        "includedsets": ["memobase:apf"]
+    }
+
+    {
+        "spec": "memobase:europeana"
+        "name": "Association of all sets defined by Memobase for OAI export to Europeana"
+        "includedsets": ["memobase:apf"]
+    }
+
+    {
+        #brauchen wir das auf Bestandesebene? - dann ein includedsets
+        "spec": "memobase:apf"
+        "name": "Association Films Plans-Fixes, all record sets"
+        "includedsets": ["memobase:apf-001"]
+        #"field": "institution"
+        #"value": "apf"
+    }
+
+    {
+        #brauchen wir das auf Bestandesebene? - dann ein includedsets
+        "spec": "memobase:apf-001"
+        "name": "Association Films Plans-Fixes, Fonds film et video Plans-Fixes"
+        "field": "recordset"
+        "value": "apf-001"
+    }
+
+
+  ]
+  metadataPrefix: [
+    {
+      "metadataPrefix": "edm"
+      "schema": "http://www.europeana.eu/schemas/edm/EDM.xsd"
+      "metadataNamespace": "https://pro.europeana.eu/page/edm-documentation"
+    }
+
+  ]
+  identify: {
+    repositoryName: "memobase OAI repositories"
+    baseURL: "https://oai.memobase.ch"
+    protocolVersion: "2.0"
+    adminEmail: "admin@memobase.ch"
+    earliestDatestamp: "2021-01-01T00:00:00.000Z"
+    deletedRecord: "transient"
+    granularity: "YYYY-MM-DDThh:mm:ss.sssZ"
+  }
+}

helm-charts/stage-secure.conf

+# Set up Play for HTTPS and locked down allowed hosts.
+# Nothing in here is required for REST, but it's a good default.
+play {
+  http {
+    cookies.strict = true
+
+    session.secure = true
+    session.httpOnly = true
+
+    flash.secure = true
+    flash.httpOnly = true
+
+    forwarded.trustedProxies = ["::1", "127.0.0.1"]
+  }
+
+  i18n {
+    langCookieSecure = true
+    langCookieHttpOnly = true
+  }
+
+  filters {
+    csrf {
+      cookie.secure = true
+    }
+
+
+    hosts {
+
+      # A list of valid hosts (e.g. "example.com") or suffixes of valid hosts (e.g. ".example.com")
+      # Note that ".example.com" will match example.com and any subdomain of example.com, with or without a trailing dot.
+      # "." matches all domains, and "" matches an empty or nonexistent host.
+      allowed = ["192.168.99.100", "localhost", ".k8s.unibas.ch", "127.0.0.1", ".memobase.ch"]
+
+      routeModifiers {
+        # If non empty, then requests will be checked if the route does not have this modifier. This is how we enable the
+        # anyhost modifier, but you may choose to use a different modifier (such as "api") if you plan to check the
+        # modifier in your code for other purposes.
+        whiteList = ["anyhost"]
+
+        # If non empty, then requests will be checked if the route contains this modifier
+        # The black list is used only if the white list is empty
+        blackList = []
+      }
+    }
+  }
+}
\ No newline at end of file

helm-charts/templates/app-config.yaml

+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-config"
+  namespace: "{{ .Values.k8sNamespace }}"
+data:
+  ELASTIC_INDEX: "{{ .Values.elasticIndex }}"
+  ELASTIC_CLUSTERNAME: "{{ .Values.elasticClustername }}"
+  RESPONSE_LISTLENGTH: "{{ .Values.responseListlength }}"
+  RESUMPTION_TOKEN_TTL: "{{ .Values.resumptionTokenTtl }}"

helm-charts/templates/deployment.yaml

+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}"
+  namespace: "{{ .Values.k8sNamespace }}"
+  labels:
+    app: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-app"
+spec:
+  selector:
+    matchLabels:
+      app: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-app"
+  replicas: {{ .Values.k8sReplicas}}
+  template:
+    metadata:
+      labels:
+        app: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-app"
+        group: "{{ .Values.k8sGroupName }}"
+        environment: "{{ .Values.k8sEnvironment }}"
+    spec:
+      containers:
+        - name: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-container"
+          image: "{{ .Values.registry }}/{{ .Values.image }}:{{ .Values.tag }}"
+          # image: guenterh/oaimemobase:latest
+          imagePullPolicy: Always
+          ports:
+            - containerPort: {{ .Values.k8sPort }}
+              name: http
+              protocol: TCP
+          resources:
+            requests:
+              cpu: "{{ .Values.k8sRequestsCpu }}"
+              memory: "{{ .Values.k8sRequestsMemory }}"
+            limits:
+              cpu: "{{ .Values.k8sLimitsCpu }}"
+              memory: "{{ .Values.k8sLimitsMemory }}"
+          volumeMounts:
+            - mountPath: "{{ .Values.oaiConfPath }}"
+              name: oaiconf
+            - mountPath: "{{ .Values.secoreConfPath }}"
+              name: secureconf
+          envFrom:
+            - configMapRef:
+                name: "{{ .Values.elasticHostConfigs }}"
+            - configMapRef:
+                name: "{{ .Values.elasticIndexConfigs }}"
+            - configMapRef:
+                name: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-config"
+      restartPolicy: Always
+      volumes:
+        - name: media-volume
+          persistentVolumeClaim:
+            claimName: {{ .Values.mediaVolumeClaimName }}
+        - name: oaiconf
+          configMap:
+            name: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment }}-oai-config"
+        - name: secureconf
+          configMap:
+            name:  "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment }}-secure-config"

k8s-manifests/oai-prod-ingress.yaml → helm-charts/templates/ingress.yaml

 apiVersion: networking.k8s.io/v1beta1
 kind: Ingress
 metadata:
-  namespace: memobase
-  name: oai-ingress
+  namespace: "{{ .Values.k8sNamespace }}"
+  name: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment }}-ingress"
   annotations:
     cert-manager.io/cluster-issuer: letsencrypt-prod
     nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
     nginx.ingress.kubernetes.io/rewrite-target: /
   labels:
-    app: oai-api
+    app: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment }}-app"
 spec:
   tls:
     - hosts:
-        - oai.memobase.k8s.unibas.ch
-      secretName: api.memobase.k8s.unibas.ch-tls
+        - "{{ .Values.k8sHost }}"
+      secretName: "{{ .Values.k8sHost }}-tls"
   rules:
-    - host: oai.memobase.k8s.unibas.ch
+    - host: "{{ .Values.k8sHost }}"
       http:
         paths:
           - path: /
             backend:
-              serviceName: oai-service
-              servicePort: 9000
+              serviceName: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment }}-service"
+              servicePort: {{ .Values.k8sPort }}

helm-charts/templates/oai-config.yaml

+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment }}-oai-config"
+  namespace: "{{ .Values.k8sNamespace }}"
+data:
+  oai.conf: |-
+    {{- (.Files.Get (printf "%s-oai.conf" .Values.k8sEnvironment)) | nindent 4 }}

helm-charts/templates/secure-config.yaml

+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment }}-secure-config"
+  namespace: "{{ .Values.k8sNamespace }}"
+data:
+  secure.conf: |-
+    {{- (.Files.Get (printf "%s-secure.conf" .Values.k8sEnvironment)) | nindent 4 }}

helm-charts/templates/service.yaml

+apiVersion: v1
+kind: Service
+metadata:
+  namespace: "{{ .Values.k8sNamespace }}"
+  name: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment }}-service"
+  labels:
+    app: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment }}-app"
+spec:
+  ports:
+    - port: {{ .Values.k8sPort }}
+  selector:
+    app: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment }}-app"
+    group: "{{ .Values.k8sGroupName }}"
+    environment: "{{ .Values.k8sEnvironment }}"
+  clusterIP: None

helm-charts/test-oai.conf

+oaiconfigs: {
+  common: {
+    xsi_schemaLocation: "http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd"
+
+  }
+  sets: [
+    {
+        "spec": "memobase"
+        "name": "Association of all sets defined by Memobase for OAI export"
+        "includedsets": ["memobase:apf"]
+    }
+
+    {
+        "spec": "memobase:europeana"
+        "name": "Association of all sets defined by Memobase for OAI export to Europeana"
+        "includedsets": ["memobase:apf"]
+    }
+
+    {
+        #brauchen wir das auf Bestandesebene? - dann ein includedsets
+        "spec": "memobase:apf"
+        "name": "Association Films Plans-Fixes, all record sets"
+        "includedsets": ["memobase:apf-001"]
+        #"field": "institution"
+        #"value": "apf"
+    }
+
+    {
+        #brauchen wir das auf Bestandesebene? - dann ein includedsets
+        "spec": "memobase:apf-001"
+        "name": "Association Films Plans-Fixes, Fonds film et video Plans-Fixes"
+        "field": "recordset"
+        "value": "apf-001"
+    }
+
+
+  ]
+  metadataPrefix: [
+    {
+      "metadataPrefix": "edm"
+      "schema": "http://www.europeana.eu/schemas/edm/EDM.xsd"
+      "metadataNamespace": "https://pro.europeana.eu/page/edm-documentation"
+    }
+
+  ]
+  identify: {
+    repositoryName: "memobase OAI repositories"
+    baseURL: "https://oai.memobase.ch"
+    protocolVersion: "2.0"
+    adminEmail: "admin@memobase.ch"
+    earliestDatestamp: "2021-01-01T00:00:00.000Z"
+    deletedRecord: "transient"
+    granularity: "YYYY-MM-DDThh:mm:ss.sssZ"
+  }
+}

helm-charts/test-secure.conf

+# Set up Play for HTTPS and locked down allowed hosts.
+# Nothing in here is required for REST, but it's a good default.
+play {
+  http {
+    cookies.strict = true
+
+    session.secure = true
+    session.httpOnly = true
+
+    flash.secure = true
+    flash.httpOnly = true
+
+    forwarded.trustedProxies = ["::1", "127.0.0.1"]
+  }
+
+  i18n {
+    langCookieSecure = true
+    langCookieHttpOnly = true
+  }
+
+  filters {
+    csrf {
+      cookie.secure = true
+    }
+
+
+    hosts {
+
+      # A list of valid hosts (e.g. "example.com") or suffixes of valid hosts (e.g. ".example.com")
+      # Note that ".example.com" will match example.com and any subdomain of example.com, with or without a trailing dot.
+      # "." matches all domains, and "" matches an empty or nonexistent host.
+      allowed = ["192.168.99.100", "localhost", ".k8s.unibas.ch", "127.0.0.1", ".memobase.ch"]
+
+      routeModifiers {
+        # If non empty, then requests will be checked if the route does not have this modifier. This is how we enable the
+        # anyhost modifier, but you may choose to use a different modifier (such as "api") if you plan to check the
+        # modifier in your code for other purposes.
+        whiteList = ["anyhost"]
+
+        # If non empty, then requests will be checked if the route contains this modifier
+        # The black list is used only if the white list is empty
+        blackList = []
+      }
+    }
+  }
+}
\ No newline at end of file