Merge branch 'gunicorn_configs'

Dockerfile

-FROM index.docker.io/library/python:3.8
+FROM index.docker.io/library/python:3.8-alpine
 
 ARG K8S_VERSION=v1.20.4
 ARG HELM_VERSION=v3.6.3
@@ -7,7 +7,7 @@ ENV FLASK_APP import_api_app
 
 EXPOSE 5000
 ENTRYPOINT ["gunicorn"]
-CMD ["--bind", "0.0.0.0:5000", "wsgi:app", "--log-level", "info", "--log-file", "/dev/stdout", "--access-logfile", "/dev/stdout"]
+CMD [ "wsgi:app", "--config", "/configs/gunicorn.conf.py"]
 
 RUN cd /usr/bin \
   && wget -q https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl \

helm-charts/configmap/prod/gunicorn.conf.py

+bind = "0.0.0.0:5000"
+reload = True
+accesslog = "/dev/stdout"
+access_log_format = '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"'
+disable_redirect_access_to_syslog = True
+errorlog = "/dev/stderr"
+loglevel = 'info'
+workers = 1
+logconfig_dict = dict(
+        version=1,
+        disable_existing_loggers=True,
+        loggers={
+            "gunicorn.error": {
+                "level": "INFO",
+                "handlers": ["error_console"],
+                "propagate": False,
+                "qualname": "gunicorn.error"
+            }
+        },
+        formatters={
+            "generic": {
+                "format": "[%(levelname)s] [%(name)s] %(message)s",
+                "datefmt": "[%Y-%m-%d %H:%M:%S %z]",
+                "class": "logging.Formatter"
+            }
+        }
+)

helm-charts/configmap/test/gunicorn.conf.py

+bind = "0.0.0.0:5000"
+reload = True
+accesslog = "/dev/stdout"
+access_log_format = '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"'
+disable_redirect_access_to_syslog = True
+errorlog = "/dev/stderr"
+loglevel = 'debug'
+workers = 1
+logconfig_dict = dict(
+        version=1,
+        disable_existing_loggers=True,
+        loggers={
+            "gunicorn.error": {
+                "level": "DEBUG",
+                "handlers": ["error_console"],
+                "propagate": False,
+                "qualname": "gunicorn.error"
+            }
+        },
+        formatters={
+            "generic": {
+                "format": "[%(levelname)s] [%(name)s] %(message)s",
+                "datefmt": "[%Y-%m-%d %H:%M:%S %z]",
+                "class": "logging.Formatter"
+            }
+        }
+)

helm-charts/helm-values/api-import-prod.yml

@@ -8,7 +8,6 @@ k8sRequestsMemory: "128Mi"
 k8sLimitsCpu: "0.4"
 k8sLimitsMemory: "256Mi"
 
-logLevel: INFO
 sftpBasePath: "/swissbib_index/mb_sftp"
 
 drupalApiCredentials: drupal-api-credentials-prod

helm-charts/helm-values/api-import-stage.yml

@@ -8,7 +8,6 @@ k8sRequestsMemory: "128Mi"
 k8sLimitsCpu: "0.4"
 k8sLimitsMemory: "256Mi"
 
-logLevel: DEBUG
 sftpBasePath: "/swissbib_index/mb_sftp/stage"
 
 drupalApiCredentials: drupal-api-credentials-stage

helm-charts/helm-values/api-import-test.yml

@@ -8,7 +8,6 @@ k8sRequestsMemory: "128Mi"
 k8sLimitsCpu: "0.4"
 k8sLimitsMemory: "256Mi"
 
-logLevel: DEBUG
 sftpBasePath: "/swissbib_index/mb_sftp/test"
 
 drupalApiCredentials: drupal-api-credentials-test

helm-charts/templates/configmap.yml

@@ -3,6 +3,10 @@ kind: ConfigMap
 metadata:
   name: {{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-config
   namespace: memobase
+  labels:
+    app: "{{ .Values.k8sName }}"
+    group: "{{ .Values.k8sGroupName }}"
+    environment: "{{ .Values.k8sEnvironment }}"
 data:
   ENV: {{ .Values.k8sEnvironment }}
   TFV_KAFKA_SERVER_CONFIGS: {{ .Values.kafkaConfigs }}
@@ -14,6 +18,5 @@ data:
   TOPIC_CONFIGS: {{ .Values.topicConfigs }}
   TOPIC_DRUPAL_EXPORT: {{ .Values.topicDrupalExport }}
   CLEAR_CACHE_URL: {{ .Values.clearCacheUrl }}
-  LOG_LEVEL: "{{ .Values.logLevel }}"
   NAMESPACE: "{{ .Values.namespace }}"
   SFTP_BASE_PATH: "{{ .Values.sftpBasePath }}"

helm-charts/templates/deployment.yml

@@ -4,24 +4,24 @@ metadata:
   name: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}"
   namespace: "{{ .Values.k8sNamespace }}"
   labels:
-    app: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-app"
+    app: "{{ .Values.k8sName }}"
     group: "{{ .Values.k8sGroupName }}"
     environment: "{{ .Values.k8sEnvironment }}"
 spec:
   selector:
     matchLabels:
-      app: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-app"
+      app: "{{ .Values.k8sName }}"
       group: "{{ .Values.k8sGroupName }}"
       environment: "{{ .Values.k8sEnvironment }}"
   replicas: {{ .Values.k8sReplicas }}
   template:
     metadata:
       labels:
-        app: {{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-app
-        group: {{ .Values.k8sGroupName }}
-        environment: {{ .Values.k8sEnvironment }}
+        app: "{{ .Values.k8sName }}"
+        group: "{{ .Values.k8sGroupName }}"
+        environment: "{{ .Values.k8sEnvironment }}"
     spec:
-      serviceAccountName: {{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-service-account #to be able to manage other pods inside the cluster
+      serviceAccountName: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-service-account"
       containers:
         - name: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-container"
           image: "{{ .Values.registry }}/{{ .Values.image }}:{{ .Values.tag }}"
@@ -34,6 +34,9 @@ spec:
                 name: "{{ .Values.kafkaConfigs }}"
             - configMapRef:
                 name: "{{ .Values.importApiConfig }}"
+          volumeMounts:
+            - name: gunicorn-conf
+              mountPath: "/configs"
           ports:
             - containerPort: 5000
               name: http
@@ -46,3 +49,7 @@ spec:
             limits:
               cpu: "{{ .Values.k8sLimitsCpu }}"
               memory: "{{ .Values.k8sLimitsMemory }}"
+      volumes:
+        - name: gunicorn-conf
+          configMap:
+            name: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment }}-gunicorn-config-map"
\ No newline at end of file

helm-charts/templates/gunicorn.configmap.yml

+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment }}-gunicorn-config-map"
+  namespace: "{{ .Values.k8sNamespace }}"
+  labels:
+    app: "{{ .Values.k8sName }}"
+    group: "{{ .Values.k8sGroupName }}"
+    environment: "{{ .Values.k8sEnvironment }}"
+data:
+  {{- if eq .Values.k8sEnvironment "test" }}
+    {{- (.Files.Glob "configmap/test/gunicorn.conf.py").AsConfig | nindent 2 }}
+  {{- else }}
+    {{- (.Files.Glob "configmap/prod/gunicorn.conf.py").AsConfig | nindent 2 }}
+  {{- end }}
\ No newline at end of file

helm-charts/templates/serviceaccount-role-rolebinding.yml

-#create service account so that this pod can manage the other pods
 apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: {{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-service-account
 ---
-#necessary role so that the service account can access the kubernetes api to list pods
 kind: Role
 apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   namespace: {{ .Values.k8sNamespace }}
   name: import-api-roles-{{ .Values.k8sEnvironment }}
 rules:
-  - apiGroups: ["", "batch"] # "" indicates the core API group
+  - apiGroups: ["", "batch"]
     resources: ["pods", "secrets", "configmaps", "jobs"]
     verbs: ["get", "list", "create", "watch", "create", "update", "patch", "delete"]
 ---

helm-charts/values.yaml

@@ -13,7 +13,6 @@ k8sRequestsMemory: placeholder
 k8sLimitsCpu: placeholder
 k8sLimitsMemory: placeholder
 
-logLevel: placeholder
 sftpBasePath: placeholder
 namespace: memobase