Commit 95ee2929 authored by Jürgen Enge's avatar Jürgen Enge
Browse files

restore from backup

parent 42b1c29c
# Streaming Server
Test Implementation for Media Streaming Server
\ No newline at end of file
Memoriav Media Streaming Server
## Installation
go get gitlab.switch.ch/memoriav/memobase-2020/streaming-server
go build gitlab.switch.ch/memoriav/memobase-2020/streaming-server/main
## Prerequsites
MariaDB/MySQL Server with table structure of [table.sql]()
\ No newline at end of file
......@@ -20,6 +20,7 @@ func (d *duration) UnmarshalText(text []byte) error {
type CfgResolverDBMySQL struct {
Dsn string
ConnMaxTimeout duration
Schema string
Query string
}
......@@ -31,6 +32,11 @@ type IIIFConfig struct {
ViewerTemplate string
}
type FileMap struct {
Alias string
Folder string
}
type Config struct {
Logfile string
Loglevel string
......@@ -45,11 +51,14 @@ type Config struct {
StaticPrefix string
IIIF IIIFConfig
JwtKey string
JwtAlg []string
Signatures map[string]memostream.Sig
ResolverDBMySQL CfgResolverDBMySQL
ResolverCacheSize int
ErrorTemplate string
VideoViewerTemplate string
AudioViewerTemplate string
FileMap []FileMap
}
func LoadConfig(filepath string) Config {
......
......@@ -11,6 +11,7 @@ import (
"os"
"os/signal"
"path/filepath"
"strings"
"syscall"
"time"
)
......@@ -85,7 +86,33 @@ func main() {
accesslog = f
}
srv := memostream.NewServer(config.Basedir, config.UrlPrefix, config.CmdPrefix, config.IIIF.Prefix, config.IIIF.Base, config.IIIF.Url, config.IIIF.JwtSubPrefix, config.IIIF.ViewerTemplate, config.Addr, resolver, config.JwtKey, log, accesslog, config.ErrorTemplate, config.VideoViewerTemplate, config.StaticPrefix, config.StaticDir, )
mapping := map[string]string{}
for _, val := range config.FileMap {
mapping[strings.ToLower(val.Alias)] = val.Folder
}
fm := memostream.NewFileMapper(mapping)
srv := memostream.NewServer(
config.Basedir,
config.UrlPrefix,
config.CmdPrefix,
config.IIIF.Prefix,
config.IIIF.Base,
config.IIIF.Url,
config.IIIF.JwtSubPrefix,
config.IIIF.ViewerTemplate,
config.Addr,
resolver,
fm,
config.JwtKey,
config.JwtAlg,
log,
accesslog,
config.ErrorTemplate,
config.VideoViewerTemplate,
config.AudioViewerTemplate,
config.StaticPrefix,
config.StaticDir, )
go func() {
if err := srv.ListenAndServe(config.CertPEM, config.KeyPEM); err != nil {
log.Errorf("server died: %v", err)
......
logfile = "" # log file location
loglevel = "DEBUG" # CRITICAL|ERROR|WARNING|NOTICE|INFO|DEBUG
accesslog = "" # http access log file
addr = "localhost:81"
addr = "localhost:82"
certpem = "" # tls client certificate file in PEM format
keypem = "" # tls client key file in PEM format
basedir = "c:/temp/"
......@@ -11,9 +11,19 @@ urlprefix = "/memo/" # prefix for accessing signature based content
# http://localhost:81/command/clearcache?auth=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJjbWQ6Y2xlYXJjYWNoZSIsImV4cCI6MTgxNjIzOTAyMn0.M_Y6R4yMAFEyo534-SXAffPwdHv929WcuSgQUcjiz10
cmdprefix = "/command/"
jwtkey = "swordfish"
jwtalg = ["HS256", "HS384", "HS512"] # "hs256" "hs384" "hs512" "es256" "es384" "es512" "ps256" "ps384" "ps512"
resolverCacheSize = 1000
errorTemplate = "C:/daten/go/src/gitlab.switch.ch/memoriav/memobase-2020/streaming-server/templates/error.gohtml" # error message for memoHandler
videoviewertemplate = "C:/daten/go/src/gitlab.switch.ch/memoriav/memobase-2020/streaming-server/templates/videojs.gohtml"
audioviewertemplate = "C:/daten/go/src/gitlab.switch.ch/memoriav/memobase-2020/streaming-server/templates/audiohowler.gohtml"
[[filemap]]
alias = "c"
folder = "c:/"
[[filemap]]
alias = "blah"
folder = "c:/temp"
[iiif]
prefix = "/iiif/"
......@@ -30,7 +40,8 @@ videoviewertemplate = "C:/daten/go/src/gitlab.switch.ch/memoriav/memobase-2020/s
# should be smaller than server connection timeout to allow controlled reconnect
connMaxTimeout = "4h"
# query has to return the fields uri, access and protocol. One parameter
query = "SELECT uri, access, proto AS protocol FROM test.test2 WHERE sig = ?"
query = "SELECT uri, access, proto AS protocol, `status` FROM test.entities WHERE sig = ?"
schema = "test"
[signatures]
[signatures.sig-01]
......
// This file is part of Memobase Mediaserver which is released under GPLv3.
// See file license.txt for full license details.
//
// Author Juergen Enge <juergen@info-age.net>
//
// This code uses elements from
// * "Mediaserver" (Center for Digital Matter HGK FHNW, Basel)
// * "Remote Exhibition Project" (info-age GmbH, Basel)
//
package memostream
import (
"errors"
"fmt"
"net/url"
"os"
"path/filepath"
"runtime"
"strings"
)
type FileMapper struct {
mapping map[string]string
}
func NewFileMapper(mapping map[string]string) *FileMapper {
return &FileMapper{mapping:mapping}
}
func (fm *FileMapper) Get(uri *url.URL) (string, error) {
if uri.Scheme != "file" {
return "", errors.New( fmt.Sprintf("cannot handle scheme %s: need file scheme", uri.Scheme))
}
var filename string
var ok bool
if uri.Host != "" {
filename, ok = fm.mapping[strings.ToLower(uri.Host)]
if !ok {
return "", errors.New(fmt.Sprintf("no mapping for %s", uri.Host))
}
}
filename = filepath.Join(filename, uri.Path)
filename = filepath.Clean(filename)
if runtime.GOOS == "windows" {
filename = strings.TrimPrefix(filename, string(os.PathSeparator))
}
return filename, nil
}
// This file is part of Memobase Mediaserver which is released under GPLv3.
// See file license.txt for full license details.
//
// Author Juergen Enge <juergen@info-age.net>
//
// This code uses elements from
// * "Mediaserver" (Center for Digital Matter HGK FHNW, Basel)
// * "Remote Exhibition Project" (info-age GmbH, Basel)
//
package memostream
import (
......@@ -27,7 +37,7 @@ func FileExists(filename string) bool {
return !info.IsDir()
}
func CheckRequestJWT(req *http.Request, secret, subject string) error {
func CheckRequestJWT(req *http.Request, secret string, alg []string, subject string) error {
var token []string
var ok bool
......@@ -50,18 +60,27 @@ func CheckRequestJWT(req *http.Request, secret, subject string) error {
return errors.New(fmt.Sprintf("Access denied: no jwt token found"))
}
}
if err := CheckJWT(token[0], secret, subject); err != nil {
if err := CheckJWT(token[0], secret, alg, subject); err != nil {
return emperror.Wrapf(err, "Access denied: token check failed")
}
return nil
}
func CheckJWT(tokenstring string, secret string, subject string) error {
func CheckJWT(tokenstring string, secret string, alg []string, subject string) error {
subject = strings.TrimRight(strings.ToLower(subject), "/")
token, err := jwt.Parse(tokenstring, func(token *jwt.Token) (interface{}, error) {
if _, ok := token.Method.(*jwt.SigningMethodHMAC); !ok {
return false, fmt.Errorf("Unexpected signing method: %v", token.Header["alg"])
talg := token.Method.Alg()
algOK := false
for _, a := range alg {
if talg == a {
algOK = true
break
}
}
if !algOK {
return false, fmt.Errorf("Unexpected signing method (allowed are %v): %v", alg, token.Header["alg"])
}
return []byte(secret), nil
})
if err != nil {
......@@ -104,9 +123,30 @@ func CreateLogger(module string, logfile string, loglevel string) (log *logging.
return
}
func NewJWT(secret string, subject string, valid int64) (tokenString string, err error) {
func NewJWT(secret string, subject string, alg string, valid int64) (tokenString string, err error) {
var signingMethod jwt.SigningMethod
switch strings.ToLower(alg) {
case "hs256":
signingMethod = jwt.SigningMethodHS256
case "hs384":
signingMethod = jwt.SigningMethodHS384
case "hs512":
signingMethod = jwt.SigningMethodHS512
case "es256":
signingMethod = jwt.SigningMethodES256
case "es384":
signingMethod = jwt.SigningMethodES384
case "es512":
signingMethod = jwt.SigningMethodES512
case "ps256":
signingMethod = jwt.SigningMethodPS256
case "ps384":
signingMethod = jwt.SigningMethodPS384
case "ps512":
signingMethod = jwt.SigningMethodPS512
}
exp := time.Now().Unix() + valid
token := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{
token := jwt.NewWithClaims(signingMethod, jwt.MapClaims{
"sub": strings.ToLower(subject),
"exp": exp,
})
......
// This file is part of Memobase Mediaserver which is released under GPLv3.
// See file license.txt for full license details.
//
// Author Juergen Enge <juergen@info-age.net>
//
// This code uses elements from
// * "Mediaserver" (Center for Digital Matter HGK FHNW, Basel)
// * "Remote Exhibition Project" (info-age GmbH, Basel)
//
package memostream
import (
"net"
"time"
)
/*
connection enhancment to make sure, that unused connections
will be closed afert timeout
*/
type IdleTimeoutConn struct {
conn net.Conn
idleTimeout time.Duration
}
func NewIdleTimeoutConn( conn net.Conn, idleTimeout time.Duration) IdleTimeoutConn {
return IdleTimeoutConn{
conn: conn,
idleTimeout: idleTimeout,
}
}
func (itc IdleTimeoutConn) Read(buf []byte) (int, error) {
itc.conn.SetDeadline(time.Now().Add(itc.idleTimeout))
return itc.conn.Read(buf)
}
func (itc IdleTimeoutConn) Write(buf []byte) (int, error) {
itc.conn.SetDeadline(time.Now().Add(itc.idleTimeout))
return itc.conn.Write(buf)
}
// This file is part of Memobase Mediaserver which is released under GPLv3.
// See file license.txt for full license details.
//
// Author Juergen Enge <juergen@info-age.net>
//
// This code uses elements from
// * "Mediaserver" (Center for Digital Matter HGK FHNW, Basel)
// * "Remote Exhibition Project" (info-age GmbH, Basel)
//
package memostream
import (
"errors"
"fmt"
"github.com/goph/emperror"
"net/url"
"path/filepath"
"runtime"
......@@ -12,6 +23,7 @@ import (
// protocol and access types need to be enums
type MediaProtocol int
type MediaAccess int
type MediaStatus int
const (
Media_File MediaProtocol = 0
......@@ -20,14 +32,18 @@ const (
Media_Public MediaAccess = 0
Media_Private MediaAccess = 1
Media_OK MediaStatus = 0
Media_Error MediaStatus = 1
Media_New = 2
)
var (
// conversion string to protocol
MediaProtocolString = map[string]MediaProtocol{
"file": Media_File,
"file": Media_File,
"redirect": Media_Redirect,
"proxy": Media_Proxy,
"proxy": Media_Proxy,
}
// conversion protocol number to string
MediaProtocolNum = map[MediaProtocol]string{
......@@ -38,7 +54,7 @@ var (
// conversion string to access type
MediaAccessString = map[string]MediaAccess{
"public": Media_Public,
"public": Media_Public,
"closed": Media_Private,
}
// conversion access number to access string
......@@ -46,6 +62,17 @@ var (
Media_Public: "public",
Media_Private: "closed",
}
MediaStatusString = map[string]MediaStatus{
"ok": Media_OK,
"error": Media_Error,
"new": Media_New,
}
MediaStatusNum = map[MediaStatus]string{
Media_OK: "ok",
Media_Error: "error",
Media_New: "new",
}
)
// Represents the data needed to stream media object
......@@ -54,10 +81,44 @@ type MediaEntry struct {
URI *url.URL
Protocol MediaProtocol
Access MediaAccess
Status MediaStatus
}
func NewMediaEntry(signature, uri, access, protocol, status string) (*MediaEntry, error) {
p, ok := MediaProtocolString[protocol]
// invalid data in database
if !ok {
return nil, errors.New(fmt.Sprintf("invalid protocol value %s", protocol))
}
a, ok := MediaAccessString[access]
// invalid data in database
if !ok {
return nil, errors.New(fmt.Sprintf("invalid access value %s", access))
}
s, ok := MediaStatusString[status]
// invalid data in database
if !ok {
return nil, errors.New(fmt.Sprintf("invalid statuss value %s", status))
}
// Create url and check uri syntax
url, err := url.Parse(uri)
if err != nil {
return nil, emperror.Wrapf(err, "cannot parse uri %s", uri)
}
return &MediaEntry{
Signature: signature,
URI: url,
Protocol: p,
Access: a,
Status: s,
}, nil
}
// get the filepath in a clean way
func (me *MediaEntry) getFilePath() (string, error) {
func (me *MediaEntry) xgetFilePath() (string, error) {
if me.URI.Scheme != "file" {
return "", errors.New(fmt.Sprintf("invalid url scheme: %s", me.URI.Scheme))
}
......
// This file is part of Memobase Mediaserver which is released under GPLv3.
// See file license.txt for full license details.
//
// Author Juergen Enge <juergen@info-age.net>
//
// This code uses elements from
// * "Mediaserver" (Center for Digital Matter HGK FHNW, Basel)
// * "Remote Exhibition Project" (info-age GmbH, Basel)
//
package memostream
import (
......@@ -33,16 +43,39 @@ type memoServer struct {
iiifJwtSubPrefix string
iiifViewerTemplate *template.Template
videoViewerTemplate *template.Template
audioViewerTemplate *template.Template
resolver *ResolverCache
host string
port string
jwtSecret string
jwtAlg []string
log *logging.Logger
accesslog io.Writer
errorTemplate *template.Template
mapping *FileMapper
}
func NewServer(basedir, urlPrefix, cmdPrefix, iiifPrefix, iiifBase, iiifUrl, iiifJwtSubPrefix, iiifViewerTemplate, addr string, resolver *ResolverCache, jwtSecret string, log *logging.Logger, accesslog io.Writer, errorTemplate, videoViewerTemplate, staticPrefix, staticDir string) *memoServer {
func NewServer(
basedir,
urlPrefix,
cmdPrefix,
iiifPrefix,
iiifBase,
iiifUrl,
iiifJwtSubPrefix,
iiifViewerTemplate,
addr string,
resolver *ResolverCache,
mapping *FileMapper,
jwtSecret string,
jwtAlg []string,
log *logging.Logger,
accesslog io.Writer,
errorTemplate,
videoViewerTemplate,
audioViewerTemplate,
staticPrefix,
staticDir string) *memoServer {
urlPrefix = "/" + strings.Trim(urlPrefix, "/") + "/"
cmdPrefix = "/" + strings.Trim(cmdPrefix, "/") + "/"
host, port, err := net.SplitHostPort(addr)
......@@ -54,6 +87,7 @@ func NewServer(basedir, urlPrefix, cmdPrefix, iiifPrefix, iiifBase, iiifUrl, iii
//mh: NewMemoHandler(baseDir, urlPrefix, resolver, jwtSecret, log, errorTemplate),
baseDir: basedir,
resolver: resolver,
mapping: mapping,
urlPrefix: urlPrefix,
cmdPrefix: cmdPrefix,
iiifPrefix: iiifPrefix,
......@@ -62,9 +96,11 @@ func NewServer(basedir, urlPrefix, cmdPrefix, iiifPrefix, iiifBase, iiifUrl, iii
iiifJwtSubPrefix: iiifJwtSubPrefix,
iiifViewerTemplate: template.Must(template.ParseFiles(iiifViewerTemplate)),
videoViewerTemplate: template.Must(template.ParseFiles(videoViewerTemplate)),
audioViewerTemplate: template.Must(template.ParseFiles(audioViewerTemplate)),
host: host,
port: port,
jwtSecret: jwtSecret,
jwtAlg: jwtAlg,
log: log,
accesslog: accesslog,
errorTemplate: template.Must(template.ParseFiles(errorTemplate)),
......@@ -145,6 +181,9 @@ func (ms *memoServer) ListenAndServe(cert, key string) error {
PathPrefix(ms.staticPrefix).
Handler(http.StripPrefix(ms.staticPrefix, http.FileServer(http.Dir(ms.staticDir))))
// metric beat handler
router.HandleFunc("/debug/vars", metricsHandler)
loggedRouter := handlers.LoggingHandler(ms.accesslog, router)
addr := net.JoinHostPort(ms.host, ms.port)
ms.srv = &http.Server{
......
// This file is part of Memobase Mediaserver which is released under GPLv3.
// See file license.txt for full license details.
//
// Author Juergen Enge <juergen@info-age.net>
//
// This code uses elements from
// * "Mediaserver" (Center for Digital Matter HGK FHNW, Basel)
// * "Remote Exhibition Project" (info-age GmbH, Basel)
//
package memostream
import (
......@@ -15,7 +25,7 @@ func (ms *memoServer) commandHandler(w http.ResponseWriter, req *http.Request) {
http.Error(w, errStr, http.StatusNotFound)
return
}
if err := CheckRequestJWT(req, ms.jwtSecret, "cmd:"+cmd); err != nil {
if err := CheckRequestJWT(req, ms.jwtSecret, ms.jwtAlg, "cmd:"+cmd); err != nil {
errStr := fmt.Sprintf("Access denied: token check failed: %v", err)
ms.log.Error(errStr)
http.Error(w, errStr, http.StatusForbidden)
......
// This file is part of Memobase Mediaserver which is released under GPLv3.
// See file license.txt for full license details.
//
// Author Juergen Enge <juergen@info-age.net>
//
// This code uses elements from
// * "Mediaserver" (Center for Digital Matter HGK FHNW, Basel)
// * "Remote Exhibition Project" (info-age GmbH, Basel)
//
package memostream
import (
......@@ -33,7 +43,7 @@ func (ms *memoServer) proxyIIIF(req *http.Request, writer http.ResponseWriter, s
newtoken := "open"
if !public {
sub := signature // ms.iiifJwtSubPrefix + filePath
newtoken, err = NewJWT(ms.jwtSecret, sub, 7200)
newtoken, err = NewJWT(ms.jwtSecret, ms.jwtAlg[0], sub, 7200)
if err != nil {
return emperror.Wrapf(err, "Error creating access token")
}
......@@ -96,14 +106,14 @@ func (ms *memoServer) HandlerIIIF(writer http.ResponseWriter, req *http.Request)
return
}
if me.Access != Media_Public {
if err := CheckJWT(token, ms.jwtSecret, signature); err != nil {
if err := CheckJWT(token, ms.jwtSecret, ms.jwtAlg, signature); err != nil {
ms.DoPanicf(writer, http.StatusForbidden, "Access denied: token check failed: %v", err)
return
}
}
sigfile, err := me.getFilePath()
sigfile, err := ms.mapping.Get(me.URI) // me.getFilePath()
if err != nil {
ms.DoPanicf(writer, http.StatusNotFound, "File for signature %s not found: %s", signature, err)
return
......
// This file is part of Memobase Mediaserver which is released under GPLv3.
// See file license.txt for full license details.
//
// Author Juergen Enge <juergen@info-age.net>
//
// This code uses elements from
// * "Mediaserver" (Center for Digital Matter HGK FHNW, Basel)
// * "Remote Exhibition Project" (info-age GmbH, Basel)
//
package memostream
import (
......@@ -27,6 +37,7 @@ func proxy(url url.URL, w http.ResponseWriter, req *http.Request) {
rp := httputil.ReverseProxy{Director: director}
// and go for it
// should we use IdleTimeoutConn? Paranoia?
rp.ServeHTTP(w, req)
}
......@@ -49,12 +60,45 @@ func (ms *memoServer) mainHandler(w http.ResponseWriter, req *http.Request) {
return
}
if me.Access != Media_Public {
if err := CheckRequestJWT(req, ms.jwtSecret, signature); err != nil {
if err := CheckRequestJWT(req, ms.jwtSecret, ms.jwtAlg, signature); err != nil {
ms.DoPanicf(w, http.StatusForbidden, "Access denied: token check failed: %v", err)
return
}
}
// todo: create better code here
if action == "audio" && params == "view" {
type vData struct {
AudioSource string
AudioTitle string
BackgroundColor string
}
newtoken := ""
if me.Access != Media_Public {
newtoken, err = NewJWT(ms.jwtSecret, signature, ms.jwtAlg[0], 7200)