Commit 8cc8dd46 authored by Juergen Enge's avatar Juergen Enge
Browse files

Merge branch 'master' of gitlab.switch.ch:memoriav/memobase-2020/services/streaming-server

parents 6d746332 c7f5f507
Pipeline #25232 passed with stages
in 2 minutes and 34 seconds
.idea
bin/
*.jp2
*.toml
/*.toml
*.mp3
poster.png
configs/memostream.toml
......
include:
- project: 'memoriav/memobase-2020/utilities/ci-templates'
file: 'docker-image/docker-image.yml'
- project: 'memoriav/memobase-2020/utilities/ci-templates'
file: 'helm-chart/helm-chart.yml'
variables:
DOCKER_TLS_CERTDIR: ""
......
This diff is collapsed.
package main
import (
"fmt"
"strings"
"github.com/BurntSushi/toml"
"gitlab.switch.ch/memoriav/memobase-2020/services/streaming-server/pkg/memostream"
"log"
......@@ -94,13 +96,42 @@ func LoadConfig(filepath string) Config {
log.Fatalln("Error on loading config: ", err)
}
dsn := os.Getenv("DSN")
if dsn != "" {
dsn, exists := os.LookupEnv("DSN")
if exists {
conf.ResolverDBMySQL.Dsn = dsn
// removed because of security issues with password in logfile
//log.Printf("DSN from Environment: [%s]", conf.ResolverDBMySQL.Dsn)
} else {
mariadb_user, exists := os.LookupEnv("MARIADB_USER")
if !exists {
log.Fatalln("No Mariadb user defined!")
}
mariadb_password, exists := os.LookupEnv("MARIADB_PASSWORD")
if !exists {
log.Fatalln("No Mariadb password defined!")
}
mariadb_host, exists := os.LookupEnv("MARIADB_HOST")
if !exists {
log.Fatalln("No Mariadb host defined!")
}
mariadb_port, exists := os.LookupEnv("MARIADB_PORT")
if !exists {
log.Fatalln("No Mariadb port defined!")
}
mariadb_database, exists := os.LookupEnv("MARIADB_DATABASE")
if !exists {
log.Fatalln("No Mariadb database defined!")
}
conf.ResolverDBMySQL.Dsn = fmt.Sprintf(
"%s:%s@tcp(%s:%s)/%s",
strings.Trim(mariadb_user, "\n"),
strings.Trim(mariadb_password, "\n"),
mariadb_host,
mariadb_port,
mariadb_database,
)
// removed because of security issues with password in logfile
//log.Printf("DSN from Config: [%s]", conf.ResolverDBMySQL.Dsn)
log.Printf("DSN from Config: [%s]", conf.ResolverDBMySQL.Dsn)
}
return conf
......
......@@ -79,6 +79,7 @@ func main() {
}
var resolverDB memostream.ResolverDB
log.Infof("DNS Resolver: %v", config.ResolverDBMySQL.Dsn)
if config.ResolverDBMySQL.Dsn == "" {
resolverDB = memostream.NewResolverDBStatic(config.Signatures)
} else {
......
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/
\ No newline at end of file
apiVersion: v2
name: streaming-server
description: A helm chart for the Memobase Streaming Server
type: application
version: 0.0.0
appVersion: 0.0.0
maintainers:
- name: Sebastian Schüpbach
email: sebastian.schuepbach@unibas.ch
tag: "latest"
k8sEnvironment: prod
k8sReplicas: 1
k8sRequestsCpu: "0.1"
k8sRequestsMemory: "128Mi"
k8sLimitsCpu: "1"
k8sLimitsMemory: "1Gi"
k8sPort: 8081
k8sMediaHost: media.memobase.k8s.unibas.ch
k8sIIIFHost: iiif.memobase.k8s.unibas.ch
mariadbDatabaseConfigs: prod-mariadb-db-medienserver-configs
mariadbUserSecrets: prod-mariadb-user-medienserver-secrets
mediaVolumeClaimName: "media-volume-claim"
mediaFolderRootPath: "/data"
tag: "latest"
k8sEnvironment: stage
k8sReplicas: 1
k8sRequestsCpu: "0.1"
k8sRequestsMemory: "128Mi"
k8sLimitsCpu: "1"
k8sLimitsMemory: "1Gi"
k8sPort: 8081
k8sMediaHost: media-stage.memobase.k8s.unibas.ch
k8sIIIFHost: iiif-stage.memobase.k8s.unibas.ch
mariadbDatabaseConfigs: stage-mariadb-db-medienserver-configs
mariadbUserSecrets: stage-mariadb-user-medienserver-secrets
mediaVolumeClaimName: stage-media-volume-claim
mediaFolderRootPath: "/data"
tag: "latest"
k8sEnvironment: test
k8sReplicas: 1
k8sRequestsCpu: "0.1"
k8sRequestsMemory: "128Mi"
k8sLimitsCpu: "1"
k8sLimitsMemory: "1Gi"
k8sPort: 8081
k8sMediaHost: "media-test.memobase.k8s.unibas.ch"
k8sIIIFHost: "iiif-test.memobase.k8s.unibas.ch"
mariadbDatabaseConfigs: test-mariadb-db-medienserver-configs
mariadbUserSecrets: test-mariadb-user-medienserver-secrets
mediaVolumeClaimName: test-media-volume-claim
mediaFolderRootPath: "/data"
logfile = "" # log file location
loglevel = "DEBUG" # CRITICAL|ERROR|WARNING|NOTICE|INFO|DEBUG
accesslog = "" # http access log file
addr = "0.0.0.0:8081"
certpem = "" # tls client certificate file in PEM format
keypem = "" # tls client key file in PEM format
staticdir = "/app/static/"
cachetimeout = "3m"
insecureproxy = true
staticprefix = "/static/"
urlprefix = "/memo/" # prefix for accessing signature based content
# http://localhost:81/command/clearcache?auth=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJjbWQ6Y2xlYXJjYWNoZSIsImV4cCI6MTgxNjIzOTAyMn0.M_Y6R4yMAFEyo534-SXAffPwdHv929WcuSgQUcjiz10
cmdprefix = "/command/"
jwtkey = "swordfish"
jwtalg = ["HS256", "HS384", "HS512"] # "hs256" "hs384" "hs512" "es256" "es384" "es512" "ps256" "ps384" "ps512"
resolverCacheSize = 1000
errorTemplate = "/app/templates/error.gohtml" # error message for memoHandler
baseurl = "https://media.memobase.k8s.unibas.ch/"
[viewer]
image = "/app/templates/iiif-image.gohtml"
video = "/app/templates/video.gohtml"
audio = "/app/templates/audio.gohtml"
srfaudio = "/app/templates/player--srg.gohtml"
srfvideo = "/app/templates/player--srg.gohtml"
vimeo = "/app/templates/video--vimeo.gohtml"
youtube = "/app/templates/video--youtube.gohtml"
[[filemap]]
alias = "testdata"
folder = "/data/"
[iiif]
prefix = "/iiif/"
base = "/data/"
url = "http://prod-imageserver-service:8182/iiif/2/"
jwtsubprefix = "iiif:"
viewertemplate = "/app/templates/openseadragon.gohtml"
manifest = "https://iiif.memobase.k8s.unibas.ch/iiif"
rendering = "https://memobase.ch"
[resolverDBMySQL]
#if dsn is empty, the static resolver will be used
#[username[:password]@][protocol[(address)]]/dbname[?param1=value1&...&paramN=valueN]
dsn="medienserver_prod:eiPh1och6Ea2uash@tcp(dd-db1.ub.unibas.ch:3306)/medienserver_prod"
# should be smaller than server connection timeout to allow controlled reconnect
connMaxTimeout = "4h"
# query has to return the fields uri, access and protocol. One parameter
# query = "SELECT uri, access, proto AS protocol, `status` FROM medienserver.entities WHERE sig = ?"
query = "SELECT `uri`, `access`, `proto` AS protocol, `status`, `type`, `mimetype`, `width`, `height`, `duration`, `manifest_v2`, `manifest_v3` FROM medienserver.entities_metadata WHERE sig = ?"
# schema = "test"
logfile = "" # log file location
loglevel = "DEBUG" # CRITICAL|ERROR|WARNING|NOTICE|INFO|DEBUG
accesslog = "" # http access log file
addr = "0.0.0.0:8081"
certpem = "" # tls client certificate file in PEM format
keypem = "" # tls client key file in PEM format
staticdir = "/app/static/"
cachetimeout = "3m"
insecureproxy = true
staticprefix = "/static/"
urlprefix = "/memo/" # prefix for accessing signature based content
# http://localhost:81/command/clearcache?auth=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJjbWQ6Y2xlYXJjYWNoZSIsImV4cCI6MTgxNjIzOTAyMn0.M_Y6R4yMAFEyo534-SXAffPwdHv929WcuSgQUcjiz10
cmdprefix = "/command/"
jwtkey = "swordfish"
jwtalg = ["HS256", "HS384", "HS512"] # "hs256" "hs384" "hs512" "es256" "es384" "es512" "ps256" "ps384" "ps512"
resolverCacheSize = 1000
errorTemplate = "/app/templates/error.gohtml" # error message for memoHandler
baseurl = "https://media-stage.memobase.k8s.unibas.ch/"
[viewer]
image = "/app/templates/iiif-image.gohtml"
video = "/app/templates/video.gohtml"
audio = "/app/templates/audio.gohtml"
srfaudio = "/app/templates/player--srg.gohtml"
srfvideo = "/app/templates/player--srg.gohtml"
vimeo = "/app/templates/video--vimeo.gohtml"
youtube = "/app/templates/video--youtube.gohtml"
[[filemap]]
alias = "testdata"
folder = "/data/"
[iiif]
prefix = "/iiif/"
base = "/data/"
url = "http://imageserver-service:8182/iiif/2/"
jwtsubprefix = "iiif:"
viewertemplate = "/app/templates/openseadragon.gohtml"
manifest = "https://iiif-stage.memobase.k8s.unibas.ch/iiif"
rendering = "https://memobase.ch"
[resolverDBMySQL]
#if dsn is empty, the static resolver will be used
#[username[:password]@][protocol[(address)]]/dbname[?param1=value1&...&paramN=valueN]
# dsn="medienserver@dd-db1.swissbib.unibas.ch:3306/medienserver_stage"
# should be smaller than server connection timeout to allow controlled reconnect
connMaxTimeout = "4h"
# query has to return the fields uri, access and protocol. One parameter
# query = "SELECT uri, access, proto AS protocol, `status` FROM medienserver.entities WHERE sig = ?"
query = "SELECT `uri`, `access`, `proto` AS protocol, `status`, `type`, `mimetype`, `width`, `height`, `duration`, `manifest_v2`, `manifest_v3` FROM medienserver_stage.entities_metadata WHERE sig = ?"
# schema = "test"
apiVersion: v1
kind: ConfigMap
metadata:
name: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment }}-config"
namespace: "{{ .Values.k8sNamespace }}"
data:
memostream.toml: |-
{{- (.Files.Get (printf "%s-config.toml" .Values.k8sEnvironment)) | nindent 4 }}
apiVersion: apps/v1
kind: Deployment
metadata:
name: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment }}"
namespace: "{{ .Values.k8sNamespace }}"
labels:
app: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment }}-app"
spec:
selector:
matchLabels:
app: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment }}-app"
replicas: {{ .Values.k8sReplicas }}
template:
metadata:
labels:
app: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment }}-app"
group: "{{ .Values.k8sGroupName }}"
environment: "{{ .Values.k8sEnvironment }}"
spec:
containers:
- name: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment }}-container"
image: "{{ .Values.registry }}/{{ .Values.image }}:{{ .Values.tag }}"
resources:
requests:
cpu: "{{ .Values.k8sRequestsCpu }}"
memory: "{{ .Values.k8sRequestsMemory }}"
limits:
cpu: "{{ .Values.k8sLimitsCpu }}"
memory: "{{ .Values.k8sLimitsMemory }}"
envFrom:
- configMapRef:
name: "{{ .Values.mariadbDatabaseConfigs }}"
env:
- name: MARIADB_PASSWORD
valueFrom:
secretKeyRef:
name: {{ .Values.mariadbUserSecrets }}
key: MARIADB_PASSWORD
- name: MARIADB_USER
valueFrom:
secretKeyRef:
name: {{ .Values.mariadbUserSecrets }}
key: MARIADB_USER
ports:
- containerPort: {{ .Values.k8sPort }}
name: http
protocol: TCP
imagePullPolicy: Always
volumeMounts:
- name: media-volume
mountPath: "{{ .Values.mediaFolderRootPath }}"
- name: config
mountPath: /app/configs
command: ["/app/app", "-cfg", "/app/configs/memostream.toml"]
restartPolicy: Always
volumes:
- name: media-volume
persistentVolumeClaim:
claimName: "{{ .Values.mediaVolumeClaimName }}"
- name: config
configMap:
name: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-config"
apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
namespace: "{{ .Values.k8sNamespace }}"
name: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment }}-iiif-ingress"
annotations:
cert-manager.io/cluster-issuer: letsencrypt-prod
nginx.ingress.kubernetes.io/enable-cors: "true"
nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
nginx.ingress.kubernetes.io/rewrite-target: /memo/$1/manifest/v$2
labels:
app: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment }}-iiif-app"
spec:
tls:
- hosts:
- "{{ .Values.k8sIIIFHost }}"
secretName: "{{ .Values.k8sIIIFHost }}-tls"
rules:
- host: "{{ .Values.k8sIIIFHost }}"
http:
paths:
- path: /iiif/([^/]+)/manifest/v(.)
backend:
serviceName: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment }}-service"
servicePort: {{ .Values.k8sPort }}
apiVersion: networking.k8s.io/v1beta1
kind: Ingress
metadata:
namespace: memobase
name: mediaserver-ingress
namespace: "{{ .Values.k8sNamespace }}"
name: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment }}-media-ingress"
annotations:
cert-manager.io/cluster-issuer: letsencrypt-prod
nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
nginx.ingress.kubernetes.io/rewrite-target: /
labels:
app: mediaserver-app
app: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment }}-media-app"
spec:
tls:
- hosts:
- media.memobase.k8s.unibas.ch
secretName: media.memobase.k8s.unibas.ch-tls
- "{{ .Values.k8sMediaHost }}"
secretName: "{{ .Values.k8sMediaHost }}-tls"
rules:
- host: media.memobase.k8s.unibas.ch
- host: "{{ .Values.k8sMediaHost }}"
http:
paths:
- path: /
backend:
serviceName: mediaserver-service
servicePort: 8081
serviceName: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment }}-service"
servicePort: {{ .Values.k8sPort }}
apiVersion: v1
kind: Service
metadata:
namespace: "{{ .Values.k8sNamespace }}"
name: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment }}-service"
labels:
app: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment }}-app"
spec:
ports:
- port: {{ .Values.k8sPort }}
selector:
app: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment }}-app"
group: "{{ .Values.k8sGroupName }}"
environment: "{{ .Values.k8sEnvironment }}"
clusterIP: None
logfile = "" # log file location
loglevel = "DEBUG" # CRITICAL|ERROR|WARNING|NOTICE|INFO|DEBUG
accesslog = "" # http access log file
addr = "0.0.0.0:8081"
certpem = "" # tls client certificate file in PEM format
keypem = "" # tls client key file in PEM format
staticdir = "/app/static/"
cachetimeout = "3m"
insecureproxy = true
staticprefix = "/static/"
urlprefix = "/memo/" # prefix for accessing signature based content
# http://localhost:81/command/clearcache?auth=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJjbWQ6Y2xlYXJjYWNoZSIsImV4cCI6MTgxNjIzOTAyMn0.M_Y6R4yMAFEyo534-SXAffPwdHv929WcuSgQUcjiz10
cmdprefix = "/command/"
jwtkey = "swordfish"
jwtalg = ["HS256", "HS384", "HS512"] # "hs256" "hs384" "hs512" "es256" "es384" "es512" "ps256" "ps384" "ps512"
resolverCacheSize = 1000
errorTemplate = "/app/templates/error.gohtml" # error message for memoHandler
baseurl = "https://media-test.memobase.k8s.unibas.ch/"
[viewer]
image = "/app/templates/iiif-image.gohtml"
video = "/app/templates/video.gohtml"
audio = "/app/templates/audio.gohtml"
srfaudio = "/app/templates/player--srg.gohtml"
srfvideo = "/app/templates/player--srg.gohtml"
vimeo = "/app/templates/video--vimeo.gohtml"
youtube = "/app/templates/video--youtube.gohtml"
[[filemap]]
alias = "testdata"
folder = "/data/"
[iiif]
prefix = "/iiif/"
base = "/data/"
url = "http://test-imageserver-service:8182/iiif/2/"
jwtsubprefix = "iiif:"
viewertemplate = "/app/templates/openseadragon.gohtml"
manifest = "https://iiif-test.memobase.k8s.unibas.ch/iiif"
rendering = "https://memobase.ch"
[resolverDBMySQL]
#if dsn is empty, the static resolver will be used
#[username[:password]@][protocol[(address)]]/dbname[?param1=value1&...&paramN=valueN]
# dsn="medienserver@mb-db1.swissbib.unibas.ch:3306/medienserver_test"
# should be smaller than server connection timeout to allow controlled reconnect
connMaxTimeout = "4h"
# query has to return the fields uri, access and protocol. One parameter
# query = "SELECT uri, access, proto AS protocol, `status` FROM medienserver.entities WHERE sig = ?"
query = "SELECT `uri`, `access`, `proto` AS protocol, `status`, `type`, `mimetype`, `width`, `height`, `duration`, `manifest_v2`, `manifest_v3` FROM medienserver_test.entities_metadata WHERE sig = ?"
# schema = "test"
registry: "cr.gitlab.switch.ch"
image: "memoriav/memobase-2020/services/streaming-server"
tag: placeholder
k8sName: media-server
k8sNamespace: memobase
k8sGroupId: api
k8sGroupName: api
k8sEnvironment: placeholder
k8sReplicas: placeholder
k8sRequestsCpu: placeholder
k8sRequestsMemory: placeholder
k8sLimitsCpu: placeholder
k8sLimitsMemory: placeholder
k8sPort: placeholder
k8sMediaHost: placeholder
k8sIIIFHost: placeholder
mariadbDatabaseConfigs: placeholder
mariadbUserSecrets: placeholder
mediaVolumeClaimName: placeholder
mediaFolderRootPath: placeholder
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
namespace: memobase
name: media-volume-claim
labels:
app: mediaserver-app
spec:
accessModes:
- ReadWriteMany
resources:
requests:
storage: 1024Gi
storageClassName: csi-cephfs
volumeMode: Filesystem
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment