Commit 9ec0d5be authored by Juergen Enge's avatar Juergen Enge
Browse files

filesystem abstraction added

parent 5d68f129
......@@ -20,7 +20,6 @@ func main() {
configFile := flag.String("cfg", "./memostream.toml", "config file location")
flag.Parse()
var exPath = ""
// if configfile not found try path of executable as prefix
if !memostream.FileExists(*configFile) {
......@@ -87,11 +86,15 @@ func main() {
accesslog = f
}
fspool := memostream.NewFilesystemPool()
mapping := map[string]string{}
for _, val := range config.FileMap {
mapping[strings.ToLower(val.Alias)] = val.Folder
}
fm := memostream.NewFileMapper(mapping)
fsd := memostream.NewFilesystemDisk(mapping)
fspool.SetFilesystem("file", fsd)
srv, err := memostream.NewServer(
config.BaseUrl,
......@@ -104,7 +107,7 @@ func main() {
config.IIIF.ViewerTemplate,
config.Addr,
resolver,
fm,
fspool,
config.JwtKey,
config.JwtAlg,
log,
......@@ -114,7 +117,7 @@ func main() {
config.AudioViewerTemplate,
config.StaticPrefix,
config.StaticDir,
)
)
if err != nil {
log.Errorf("error initializing server: %v", err)
return
......
......@@ -35,13 +35,14 @@ folder = "C:/temp"
[resolverDBMySQL]
#if dsn is empty, the static resolver will be used
#[username[:password]@][protocol[(address)]]/dbname[?param1=value1&...&paramN=valueN]
#dsn = "test:HDNQiaqNqu4IjmUPATJr@tcp(localhost:3306)/test"
dsn = "urlchecker:siex5ieNguuQuei@tcp(localhost:4306)/medienserver"
dsn = "test:HDNQiaqNqu4IjmUPATJr@tcp(localhost:3306)/test"
#dsn = "urlchecker:siex5ieNguuQuei@tcp(localhost:4306)/medienserver"
#dsn = ""
# should be smaller than server connection timeout to allow controlled reconnect
connMaxTimeout = "4h"
# query has to return the fields uri, access and protocol. One parameter
query = "SELECT uri, access, proto AS protocol, `status` FROM medianserver.entities WHERE sig = ?"
#query = "SELECT uri, access, proto AS protocol, `status` FROM mediaserver.entities WHERE sig = ?"
query = "SELECT uri, access, proto AS protocol, `status` FROM test.entities WHERE sig = ?"
schema = "test"
[signatures]
......
package memostream
import (
"fmt"
"io"
"net/url"
"time"
)
// io.ReadCloser with io.seeker for http.ServeContent
type ReadSeekerCloser interface {
io.ReadCloser
io.Seeker
}
// abstraction interface for creating ReasSeekerCloser
type Filesystem interface {
// creates ReadSeekerCloser, returns last modification time
Open(uri url.URL) (ReadSeekerCloser, time.Time, error)
// checks existence of uri object
Exist(uri url.URL) bool
// builds native access path
Truename(uri url.URL) string
}
// pool of filesystems
type FilesystemPool struct {
// fspool from uri-scheme to Filesystem
fss map[string]Filesystem
}
// constructor of FilesystemPool
func NewFilesystemPool() *FilesystemPool {
return &FilesystemPool{fss: map[string]Filesystem{}}
}
// set a Filesystem for a specified uri scheme
func (fp *FilesystemPool) SetFilesystem(scheme string, fs Filesystem) {
fp.fss[scheme] = fs
}
func (fp *FilesystemPool) Truename(uri url.URL) string {
fs, ok := fp.fss[uri.Scheme]
if !ok {
return "invalid"
}
return fs.Truename(uri)
}
func (fp *FilesystemPool) Exist(uri url.URL) bool {
fs, ok := fp.fss[uri.Scheme]
if !ok {
return false
}
return fs.Exist(uri)
}
// creates a ReadSeekerCloser for a specified uri from matching Filesystem
func (fp *FilesystemPool) OpenUri(uri url.URL) (ReadSeekerCloser, time.Time, error) {
fs, ok := fp.fss[uri.Scheme]
if !ok {
return nil, time.Now(), fmt.Errorf("unknown uri scheme %s from url %s", uri.Scheme, uri.String())
}
return fs.Open(uri)
}
......@@ -13,31 +13,37 @@ package memostream
import (
"errors"
"fmt"
"github.com/goph/emperror"
"net/url"
"os"
"path/filepath"
"runtime"
"strings"
"time"
)
type FileMapper struct {
// instance of Filesystem for "normal" disk filesystems
type FilesystemDisk struct {
// directory mapping structure
mapping map[string]string
}
func NewFileMapper(mapping map[string]string) *FileMapper {
return &FileMapper{mapping: mapping}
// constructor for FilesystemDisk
func NewFilesystemDisk(mapping map[string]string) *FilesystemDisk {
return &FilesystemDisk{mapping: mapping}
}
func (fm *FileMapper) Get(uri *url.URL) (string, error) {
// mapping from uri to path
func (fm *FilesystemDisk) mapUri(uri url.URL) (string, error) {
if uri.Scheme != "file" {
return "", errors.New( fmt.Sprintf("cannot handle scheme %s: need file scheme", uri.Scheme))
return "", errors.New(fmt.Sprintf("cannot handle scheme %s: need file scheme", uri.Scheme))
}
var filename string
var ok bool
if uri.Host != "" {
filename, ok = fm.mapping[strings.ToLower(uri.Host)]
if !ok {
return "", errors.New(fmt.Sprintf("no mapping for %s", uri.Host))
return "", errors.New(fmt.Sprintf("no fspool for %s", uri.Host))
}
}
filename = filepath.Join(filename, uri.Path)
......@@ -47,3 +53,41 @@ func (fm *FileMapper) Get(uri *url.URL) (string, error) {
}
return filename, nil
}
// builds native access path
func (fm *FilesystemDisk) Truename(uri url.URL) string {
path, err := fm.mapUri(uri)
if err != nil {
return "invalid"
}
return filepath.ToSlash(filepath.Clean(path))
}
// checks existence of file
func (fm *FilesystemDisk) Exist(uri url.URL) bool {
path, err := fm.mapUri(uri)
if err != nil {
return false
}
if _, err := os.Stat(path); err != nil {
return false
}
return true
}
// creates ReadSeekerCloser, returns last modification time
func (fm *FilesystemDisk) Open(uri url.URL) (ReadSeekerCloser, time.Time, error) {
path, err := fm.mapUri(uri)
if err != nil {
return nil, time.Now(), emperror.Wrapf(err, "cannot mapUri filepath from url %s", uri)
}
f, err := os.Open(path)
if err != nil {
return nil, time.Now(), emperror.Wrapf(err, "cannot open file %s", path)
}
stat, err := f.Stat()
if err != nil {
return nil, time.Now(), emperror.Wrapf(err, "cannot stat %s", path)
}
return f, stat.ModTime(), nil
}
......@@ -17,6 +17,7 @@ import (
"github.com/goph/emperror"
"github.com/op/go-logging"
"net/http"
"net/url"
"os"
"strings"
"time"
......@@ -36,6 +37,16 @@ func FileExists(filename string) bool {
return !info.IsDir()
}
// extracts last part of uri as name
func NameFromUri(uri url.URL) string {
parts := strings.Split(uri.Path, "/")
l := len(parts)
if l == 0 {
return "invalid"
}
return parts[l-1]
}
func CheckRequestJWT(req *http.Request, secret string, alg []string, subject string) error {
var token []string
var ok bool
......@@ -87,7 +98,7 @@ func CheckJWT(tokenstring string, secret string, alg []string, subject string) e
}
if claims, ok := token.Claims.(jwt.MapClaims); ok && token.Valid {
if !ok {
return fmt.Errorf("Cannot get claims from token [sub:%s]", subject)
return fmt.Errorf("Cannot mapUri claims from token [sub:%s]", subject)
}
if strings.ToLower(claims["sub"].(string)) == subject {
return nil
......@@ -160,7 +171,7 @@ func NewJWT(secret string, subject string, alg string, valid int64, domain strin
}
token := jwt.NewWithClaims(signingMethod, claims)
// log.Println("NewJWT( ", secret, ", ", subject, ", ", exp)
// log.Println("NewJWT( ", secret, ", ", subject, ", ", exp)
tokenString, err = token.SignedString([]byte(secret))
return tokenString, err
}
......
......@@ -78,7 +78,7 @@ var (
// Represents the data needed to stream media object
type MediaEntry struct {
Signature string
URI *url.URL
URI url.URL
Protocol MediaProtocol
Access MediaAccess
Status MediaStatus
......@@ -108,16 +108,19 @@ func NewMediaEntry(signature, uri, access, protocol, status string) (*MediaEntry
if err != nil {
return nil, emperror.Wrapf(err, "cannot parse uri %s", uri)
}
if url == nil {
return nil, emperror.Wrapf(err, "url from uri is nil %s", uri)
}
return &MediaEntry{
Signature: signature,
URI: url,
URI: *url,
Protocol: p,
Access: a,
Status: s,
}, nil
}
// get the filepath in a clean way
// mapUri the filepath in a clean way
func (me *MediaEntry) xgetFilePath() (string, error) {
if me.URI.Scheme != "file" {
return "", errors.New(fmt.Sprintf("invalid url scheme: %s", me.URI.Scheme))
......
......@@ -53,7 +53,7 @@ type memoServer struct {
log *logging.Logger
accesslog io.Writer
errorTemplate *template.Template
mapping *FileMapper
fspool *FilesystemPool
}
func NewServer(
......@@ -67,7 +67,7 @@ func NewServer(
iiifViewerTemplate,
addr string,
resolver *ResolverCache,
mapping *FileMapper,
mapping *FilesystemPool,
jwtSecret string,
jwtAlg []string,
log *logging.Logger,
......@@ -105,7 +105,7 @@ func NewServer(
//mh: NewMemoHandler(baseDir, urlPrefix, resolver, jwtSecret, log, errorTemplate),
baseUrl: baseUrl,
resolver: resolver,
mapping: mapping,
fspool: mapping,
urlPrefix: urlPrefix,
cmdPrefix: cmdPrefix,
iiifPrefix: iiifPrefix,
......@@ -169,6 +169,7 @@ func (ms *memoServer) ListenAndServe(cert, key string) error {
return true
}).HandlerFunc(ms.mainHandler).Methods("GET", "HEAD")
// router.PathPrefix(ms.urlPrefix).HandlerFunc(ms.mainHandler)
router.HandleFunc(ms.cmdPrefix+"{cmd}", ms.commandHandler)
// route for IIIF
......
......@@ -118,14 +118,14 @@ func (ms *memoServer) HandlerIIIF(writer http.ResponseWriter, req *http.Request)
}
}
sigfile, err := ms.mapping.Get(me.URI) // me.getFilePath()
if err != nil {
ms.DoPanicf(writer, http.StatusNotFound, "File for signature %s not found: %s", signature, err)
if !ms.fspool.Exist(me.URI) {
ms.DoPanicf(writer, http.StatusNotFound, "file not found: %v", me.URI)
return
}
sigfile := ms.fspool.Truename(me.URI) // me.getFilePath()
// create a correct filepath with slashes
filename := filepath.Clean(filepath.Join(ms.iiifBase, strings.Replace(file, "$", "/", -1)))
filename := filepath.ToSlash(filepath.Clean(filepath.Join(ms.iiifBase, strings.Replace(file, "$", "/", -1))))
if sigfile != filename {
ms.DoPanicf(writer, http.StatusForbidden, "Don't cheat!! signature %s does not match path %s", signature, filename)
return
......
......@@ -16,7 +16,6 @@ import (
"net/http"
"net/http/httputil"
"net/url"
"os"
"path/filepath"
"strings"
)
......@@ -93,8 +92,8 @@ func (ms *memoServer) mainHandler(w http.ResponseWriter, req *http.Request) {
strings.Trim(ms.urlPrefix, "/"),
signature,
newtoken),
AudioTitle: "Audio",
BackgroundColor:"#000000",
AudioTitle: "Audio",
BackgroundColor: "#000000",
}
ms.audioViewerTemplate.Execute(w, data)
return
......@@ -155,19 +154,17 @@ func (ms *memoServer) mainHandler(w http.ResponseWriter, req *http.Request) {
}
data := vData{
BackgroundColor: "#000000",
TileSource: tileSource,
TileSource: tileSource,
}
ms.iiifViewerTemplate.Execute(w, data)
return
}
if action == "iiif" && params == "info.json" {
file, err := ms.mapping.Get(me.URI) // me.getFilePath()
if err != nil {
ms.DoPanicf(w, http.StatusConflict, "no file in lookup for signature %s: %v", signature, err)
if !ms.fspool.Exist(me.URI) {
ms.DoPanicf(w, http.StatusNotFound, "file not found: %v", me.URI)
return
}
if err := ms.proxyIIIF(req, w, signature, filepath.ToSlash(file), params, me.Access == Media_Public); err != nil {
if err := ms.proxyIIIF(req, w, signature, filepath.ToSlash(ms.fspool.Truename(me.URI)), params, me.Access == Media_Public); err != nil {
ms.DoPanicf(w, http.StatusInternalServerError, "cannot proxy to iiif server: %v", err)
return
}
......@@ -176,30 +173,18 @@ func (ms *memoServer) mainHandler(w http.ResponseWriter, req *http.Request) {
switch me.Protocol {
case Media_File: // stream the file directly
path, err := ms.mapping.Get(me.URI) //me.getFilePath()
rsc, modTime, err := ms.fspool.OpenUri(me.URI)
if err != nil {
ms.DoPanicf(w, http.StatusInternalServerError, "cannot get filepath from url %ms for signature %ms", me.URI, signature)
return
}
_, err = os.Stat(path)
if os.IsNotExist(err) {
// if file not exist send 404
ms.DoPanicf(w, http.StatusNotFound, "file not found: %s", path)
http.Error(w, err.Error(), http.StatusNotFound)
return
} else if err != nil {
// something strange happened
ms.DoPanicf(w, http.StatusInternalServerError, "cannot stat file %ms: %v", path, err)
ms.DoPanicf(w, http.StatusNotFound, "cannot create reader from url %s for signature %s: %v", me.URI.String(), signature, err)
return
}
// deliver static content
http.ServeFile(w, req, path)
defer rsc.Close()
http.ServeContent(w, req, NameFromUri(me.URI), modTime, rsc)
case Media_Redirect: // just redirect
http.Redirect(w, req, me.URI.String(),
http.StatusTemporaryRedirect)
case Media_Proxy: // do all the proxy stuff
proxy(*me.URI, w, req)
proxy(me.URI, w, req)
default: // don't know what to do
ms.DoPanicf(w, http.StatusNotImplemented, "only type \"file\", \"redirect\" and \"proxy\" is supported for signature %v: type %v given", signature, me.getProtocolString())
return
......
......@@ -52,7 +52,7 @@ func (rdm *ResolverDBMySQL) Resolve(signature string) (*MediaEntry, error) {
// use prepared statement for speed
row := rdm.preparedStmt.QueryRow(signature)
var access, protocol, uri, status string
// get data
// mapUri data
switch err := row.Scan(&uri, &access, &protocol, &status); err {
case sql.ErrNoRows: // dataset not found
return nil, emperror.Wrapf(err, "cannot find signature %s", signature)
......
......@@ -19,7 +19,7 @@ import (
/**
Static resolver for testing the memoServer functionality
*/
*/
type ResolverDBStatic struct {
signatures map[string]Sig
}
......@@ -40,6 +40,9 @@ func (rds *ResolverDBStatic) Resolve(signature string) (*MediaEntry, error) {
if err != nil {
return nil, emperror.Wrapf(err, "cannot parse url %s of signature %s", sig.Uri, signature)
}
if u == nil {
return nil, emperror.Wrapf(err, "cannot parse url %s of signature %s", sig.Uri, signature)
}
p, ok := MediaProtocolString[sig.Type]
if !ok {
return nil, errors.New(fmt.Sprintf("invalid protocol %s for signature %s", sig.Type, signature))
......@@ -50,7 +53,7 @@ func (rds *ResolverDBStatic) Resolve(signature string) (*MediaEntry, error) {
}
return &MediaEntry{
Signature: signature,
URI: u,
URI: *u,
Protocol: p,
Access: a,
}, nil
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment