filesystem abstraction added

cmd/server/main.go

@@ -20,7 +20,6 @@ func main() {
 	configFile := flag.String("cfg", "./memostream.toml", "config file location")
 	flag.Parse()
 
-
 	var exPath = ""
 	// if configfile not found try path of executable as prefix
 	if !memostream.FileExists(*configFile) {
@@ -87,11 +86,15 @@ func main() {
 		accesslog = f
 	}
 
+	fspool := memostream.NewFilesystemPool()
+
 	mapping := map[string]string{}
 	for _, val := range config.FileMap {
 		mapping[strings.ToLower(val.Alias)] = val.Folder
 	}
-	fm := memostream.NewFileMapper(mapping)
+	fsd := memostream.NewFilesystemDisk(mapping)
+
+	fspool.SetFilesystem("file", fsd)
 
 	srv, err := memostream.NewServer(
 		config.BaseUrl,
@@ -104,7 +107,7 @@ func main() {
 		config.IIIF.ViewerTemplate,
 		config.Addr,
 		resolver,
-		fm,
+		fspool,
 		config.JwtKey,
 		config.JwtAlg,
 		log,
@@ -114,7 +117,7 @@ func main() {
 		config.AudioViewerTemplate,
 		config.StaticPrefix,
 		config.StaticDir,
-		)
+	)
 	if err != nil {
 		log.Errorf("error initializing server: %v", err)
 		return

configs/memostream.toml

@@ -35,13 +35,14 @@ folder = "C:/temp"
 [resolverDBMySQL]
     #if dsn is empty, the static resolver will be used
     #[username[:password]@][protocol[(address)]]/dbname[?param1=value1&...&paramN=valueN]
-    #dsn = "test:HDNQiaqNqu4IjmUPATJr@tcp(localhost:3306)/test"
-    dsn = "urlchecker:siex5ieNguuQuei@tcp(localhost:4306)/medienserver"
+    dsn = "test:HDNQiaqNqu4IjmUPATJr@tcp(localhost:3306)/test"
+    #dsn = "urlchecker:siex5ieNguuQuei@tcp(localhost:4306)/medienserver"
     #dsn = ""
     # should be smaller than server connection timeout to allow controlled reconnect
     connMaxTimeout = "4h"
     # query has to return the fields uri, access and protocol. One parameter
-    query = "SELECT uri, access, proto AS protocol, `status` FROM medianserver.entities WHERE sig = ?"
+    #query = "SELECT uri, access, proto AS protocol, `status` FROM mediaserver.entities WHERE sig = ?"
+    query = "SELECT uri, access, proto AS protocol, `status` FROM test.entities WHERE sig = ?"
     schema = "test"
 
 [signatures]

pkg/memostream/filesystem.go

+package memostream
+
+import (
+	"fmt"
+	"io"
+	"net/url"
+	"time"
+)
+
+// io.ReadCloser with io.seeker for http.ServeContent
+type ReadSeekerCloser interface {
+	io.ReadCloser
+	io.Seeker
+}
+
+// abstraction interface for creating ReasSeekerCloser
+type Filesystem interface {
+	// creates ReadSeekerCloser, returns last modification time
+	Open(uri url.URL) (ReadSeekerCloser, time.Time, error)
+
+	// checks existence of uri object
+	Exist(uri url.URL) bool
+
+	// builds native access path
+	Truename(uri url.URL) string
+}
+
+// pool of filesystems
+type FilesystemPool struct {
+	// fspool from uri-scheme to Filesystem
+	fss map[string]Filesystem
+}
+
+// constructor of FilesystemPool
+func NewFilesystemPool() *FilesystemPool {
+	return &FilesystemPool{fss: map[string]Filesystem{}}
+}
+
+// set a Filesystem for a specified uri scheme
+func (fp *FilesystemPool) SetFilesystem(scheme string, fs Filesystem) {
+	fp.fss[scheme] = fs
+}
+
+func (fp *FilesystemPool) Truename(uri url.URL) string {
+	fs, ok := fp.fss[uri.Scheme]
+	if !ok {
+		return "invalid"
+	}
+	return fs.Truename(uri)
+}
+
+func (fp *FilesystemPool) Exist(uri url.URL) bool {
+	fs, ok := fp.fss[uri.Scheme]
+	if !ok {
+		return false
+	}
+	return fs.Exist(uri)
+}
+
+// creates a ReadSeekerCloser for a specified uri from matching Filesystem
+func (fp *FilesystemPool) OpenUri(uri url.URL) (ReadSeekerCloser, time.Time, error) {
+	fs, ok := fp.fss[uri.Scheme]
+	if !ok {
+		return nil, time.Now(), fmt.Errorf("unknown uri scheme %s from url %s", uri.Scheme, uri.String())
+	}
+	return fs.Open(uri)
+}

pkg/memostream/fileMapper.go → pkg/memostream/filesystemDisk.go

@@ -13,31 +13,37 @@ package memostream
 import (
 	"errors"
 	"fmt"
+	"github.com/goph/emperror"
 	"net/url"
 	"os"
 	"path/filepath"
 	"runtime"
 	"strings"
+	"time"
 )
 
-type FileMapper struct {
+// instance of Filesystem for "normal" disk filesystems
+type FilesystemDisk struct {
+	// directory mapping structure
 	mapping map[string]string
 }
 
-func NewFileMapper(mapping map[string]string) *FileMapper {
-	return &FileMapper{mapping: mapping}
+// constructor for FilesystemDisk
+func NewFilesystemDisk(mapping map[string]string) *FilesystemDisk {
+	return &FilesystemDisk{mapping: mapping}
 }
 
-func (fm *FileMapper) Get(uri *url.URL) (string, error) {
+// mapping from uri to path
+func (fm *FilesystemDisk) mapUri(uri url.URL) (string, error) {
 	if uri.Scheme != "file" {
-		return "", errors.New( fmt.Sprintf("cannot handle scheme %s: need file scheme", uri.Scheme))
+		return "", errors.New(fmt.Sprintf("cannot handle scheme %s: need file scheme", uri.Scheme))
 	}
 	var filename string
 	var ok bool
 	if uri.Host != "" {
 		filename, ok = fm.mapping[strings.ToLower(uri.Host)]
 		if !ok {
-			return "", errors.New(fmt.Sprintf("no mapping for %s", uri.Host))
+			return "", errors.New(fmt.Sprintf("no fspool for %s", uri.Host))
 		}
 	}
 	filename = filepath.Join(filename, uri.Path)
@@ -47,3 +53,41 @@ func (fm *FileMapper) Get(uri *url.URL) (string, error) {
 	}
 	return filename, nil
 }
+
+// builds native access path
+func (fm *FilesystemDisk) Truename(uri url.URL) string {
+	path, err := fm.mapUri(uri)
+	if err != nil {
+		return "invalid"
+	}
+	return filepath.ToSlash(filepath.Clean(path))
+}
+
+// checks existence of file
+func (fm *FilesystemDisk) Exist(uri url.URL) bool {
+	path, err := fm.mapUri(uri)
+	if err != nil {
+		return false
+	}
+	if _, err := os.Stat(path); err != nil {
+		return false
+	}
+	return true
+}
+
+// creates ReadSeekerCloser, returns last modification time
+func (fm *FilesystemDisk) Open(uri url.URL) (ReadSeekerCloser, time.Time, error) {
+	path, err := fm.mapUri(uri)
+	if err != nil {
+		return nil, time.Now(), emperror.Wrapf(err, "cannot mapUri filepath from url %s", uri)
+	}
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, time.Now(), emperror.Wrapf(err, "cannot open file %s", path)
+	}
+	stat, err := f.Stat()
+	if err != nil {
+		return nil, time.Now(), emperror.Wrapf(err, "cannot stat %s", path)
+	}
+	return f, stat.ModTime(), nil
+}

pkg/memostream/helper.go

@@ -17,6 +17,7 @@ import (
 	"github.com/goph/emperror"
 	"github.com/op/go-logging"
 	"net/http"
+	"net/url"
 	"os"
 	"strings"
 	"time"
@@ -36,6 +37,16 @@ func FileExists(filename string) bool {
 	return !info.IsDir()
 }
 
+// extracts last part of uri as name
+func NameFromUri(uri url.URL) string {
+	parts := strings.Split(uri.Path, "/")
+	l := len(parts)
+	if l == 0 {
+		return "invalid"
+	}
+	return parts[l-1]
+}
+
 func CheckRequestJWT(req *http.Request, secret string, alg []string, subject string) error {
 	var token []string
 	var ok bool
@@ -87,7 +98,7 @@ func CheckJWT(tokenstring string, secret string, alg []string, subject string) e
 	}
 	if claims, ok := token.Claims.(jwt.MapClaims); ok && token.Valid {
 		if !ok {
-			return fmt.Errorf("Cannot get claims from token [sub:%s]", subject)
+			return fmt.Errorf("Cannot mapUri claims from token [sub:%s]", subject)
 		}
 		if strings.ToLower(claims["sub"].(string)) == subject {
 			return nil
@@ -160,7 +171,7 @@ func NewJWT(secret string, subject string, alg string, valid int64, domain strin
 	}
 
 	token := jwt.NewWithClaims(signingMethod, claims)
-//	log.Println("NewJWT( ", secret, ", ", subject, ", ", exp)
+	//	log.Println("NewJWT( ", secret, ", ", subject, ", ", exp)
 	tokenString, err = token.SignedString([]byte(secret))
 	return tokenString, err
 }

pkg/memostream/mediaEntry.go

@@ -78,7 +78,7 @@ var (
 // Represents the data needed to stream media object
 type MediaEntry struct {
 	Signature string
-	URI       *url.URL
+	URI       url.URL
 	Protocol  MediaProtocol
 	Access    MediaAccess
 	Status    MediaStatus
@@ -108,16 +108,19 @@ func NewMediaEntry(signature, uri, access, protocol, status string) (*MediaEntry
 	if err != nil {
 		return nil, emperror.Wrapf(err, "cannot parse uri %s", uri)
 	}
+	if url == nil {
+		return nil, emperror.Wrapf(err, "url from uri is nil %s", uri)
+	}
 	return &MediaEntry{
 		Signature: signature,
-		URI:       url,
+		URI:       *url,
 		Protocol:  p,
 		Access:    a,
 		Status:    s,
 	}, nil
 }
 
-// get the filepath in a clean way
+// mapUri the filepath in a clean way
 func (me *MediaEntry) xgetFilePath() (string, error) {
 	if me.URI.Scheme != "file" {
 		return "", errors.New(fmt.Sprintf("invalid url scheme: %s", me.URI.Scheme))

pkg/memostream/memoServer.go

@@ -53,7 +53,7 @@ type memoServer struct {
 	log                 *logging.Logger
 	accesslog           io.Writer
 	errorTemplate       *template.Template
-	mapping             *FileMapper
+	fspool              *FilesystemPool
 }
 
 func NewServer(
@@ -67,7 +67,7 @@ func NewServer(
 	iiifViewerTemplate,
 	addr string,
 	resolver *ResolverCache,
-	mapping *FileMapper,
+	mapping *FilesystemPool,
 	jwtSecret string,
 	jwtAlg []string,
 	log *logging.Logger,
@@ -105,7 +105,7 @@ func NewServer(
 		//mh:            NewMemoHandler(baseDir, urlPrefix, resolver, jwtSecret, log, errorTemplate),
 		baseUrl:             baseUrl,
 		resolver:            resolver,
-		mapping:             mapping,
+		fspool:              mapping,
 		urlPrefix:           urlPrefix,
 		cmdPrefix:           cmdPrefix,
 		iiifPrefix:          iiifPrefix,
@@ -169,6 +169,7 @@ func (ms *memoServer) ListenAndServe(cert, key string) error {
 			return true
 		}).HandlerFunc(ms.mainHandler).Methods("GET", "HEAD")
 	//	router.PathPrefix(ms.urlPrefix).HandlerFunc(ms.mainHandler)
+
 	router.HandleFunc(ms.cmdPrefix+"{cmd}", ms.commandHandler)
 
 	// route for IIIF

pkg/memostream/memoServerHandlerIIIF.go

@@ -118,14 +118,14 @@ func (ms *memoServer) HandlerIIIF(writer http.ResponseWriter, req *http.Request)
 		}
 	}
 
-
-	sigfile, err := ms.mapping.Get(me.URI) // me.getFilePath()
-	if err != nil {
-		ms.DoPanicf(writer, http.StatusNotFound, "File for signature %s not found: %s", signature, err)
+	if !ms.fspool.Exist(me.URI) {
+		ms.DoPanicf(writer, http.StatusNotFound, "file not found: %v", me.URI)
 		return
 	}
+	sigfile := ms.fspool.Truename(me.URI) // me.getFilePath()
+
 	// create a correct filepath with slashes
-	filename := filepath.Clean(filepath.Join(ms.iiifBase, strings.Replace(file, "$", "/", -1)))
+	filename := filepath.ToSlash(filepath.Clean(filepath.Join(ms.iiifBase, strings.Replace(file, "$", "/", -1))))
 	if sigfile != filename {
 		ms.DoPanicf(writer, http.StatusForbidden, "Don't cheat!! signature %s does not match path %s", signature, filename)
 		return

pkg/memostream/memoServerHandlerMain.go

@@ -16,7 +16,6 @@ import (
 	"net/http"
 	"net/http/httputil"
 	"net/url"
-	"os"
 	"path/filepath"
 	"strings"
 )
@@ -93,8 +92,8 @@ func (ms *memoServer) mainHandler(w http.ResponseWriter, req *http.Request) {
 				strings.Trim(ms.urlPrefix, "/"),
 				signature,
 				newtoken),
-			AudioTitle: "Audio",
-			BackgroundColor:"#000000",
+			AudioTitle:      "Audio",
+			BackgroundColor: "#000000",
 		}
 		ms.audioViewerTemplate.Execute(w, data)
 		return
@@ -155,19 +154,17 @@ func (ms *memoServer) mainHandler(w http.ResponseWriter, req *http.Request) {
 		}
 		data := vData{
 			BackgroundColor: "#000000",
-			TileSource: tileSource,
-
+			TileSource:      tileSource,
 		}
 		ms.iiifViewerTemplate.Execute(w, data)
 		return
 	}
 	if action == "iiif" && params == "info.json" {
-		file, err := ms.mapping.Get(me.URI) // me.getFilePath()
-		if err != nil {
-			ms.DoPanicf(w, http.StatusConflict, "no file in lookup for signature %s: %v", signature, err)
+		if !ms.fspool.Exist(me.URI) {
+			ms.DoPanicf(w, http.StatusNotFound, "file not found: %v", me.URI)
 			return
 		}
-		if err := ms.proxyIIIF(req, w, signature, filepath.ToSlash(file), params, me.Access == Media_Public); err != nil {
+		if err := ms.proxyIIIF(req, w, signature, filepath.ToSlash(ms.fspool.Truename(me.URI)), params, me.Access == Media_Public); err != nil {
 			ms.DoPanicf(w, http.StatusInternalServerError, "cannot proxy to iiif server: %v", err)
 			return
 		}
@@ -176,30 +173,18 @@ func (ms *memoServer) mainHandler(w http.ResponseWriter, req *http.Request) {
 
 	switch me.Protocol {
 	case Media_File: // stream the file directly
-		path, err := ms.mapping.Get(me.URI) //me.getFilePath()
+		rsc, modTime, err := ms.fspool.OpenUri(me.URI)
 		if err != nil {
-			ms.DoPanicf(w, http.StatusInternalServerError, "cannot get filepath from url %ms for signature %ms", me.URI, signature)
-			return
-		}
-
-		_, err = os.Stat(path)
-		if os.IsNotExist(err) {
-			// if file not exist send 404
-			ms.DoPanicf(w, http.StatusNotFound, "file not found: %s", path)
-			http.Error(w, err.Error(), http.StatusNotFound)
-			return
-		} else if err != nil {
-			// something strange happened
-			ms.DoPanicf(w, http.StatusInternalServerError, "cannot stat file %ms: %v", path, err)
+			ms.DoPanicf(w, http.StatusNotFound, "cannot create reader from url %s for signature %s: %v", me.URI.String(), signature, err)
 			return
 		}
-		// deliver static content
-		http.ServeFile(w, req, path)
+		defer rsc.Close()
+		http.ServeContent(w, req, NameFromUri(me.URI), modTime, rsc)
 	case Media_Redirect: // just redirect
 		http.Redirect(w, req, me.URI.String(),
 			http.StatusTemporaryRedirect)
 	case Media_Proxy: // do all the proxy stuff
-		proxy(*me.URI, w, req)
+		proxy(me.URI, w, req)
 	default: // don't know what to do
 		ms.DoPanicf(w, http.StatusNotImplemented, "only type \"file\", \"redirect\" and \"proxy\" is supported for signature %v: type %v given", signature, me.getProtocolString())
 		return

pkg/memostream/resolverDBMySQL.go

@@ -52,7 +52,7 @@ func (rdm *ResolverDBMySQL) Resolve(signature string) (*MediaEntry, error) {
 	// use prepared statement for speed
 	row := rdm.preparedStmt.QueryRow(signature)
 	var access, protocol, uri, status string
-	// get data
+	// mapUri data
 	switch err := row.Scan(&uri, &access, &protocol, &status); err {
 	case sql.ErrNoRows: // dataset not found
 		return nil, emperror.Wrapf(err, "cannot find signature %s", signature)

pkg/memostream/resolverDBStatic.go

@@ -19,7 +19,7 @@ import (
 
 /**
 Static resolver for testing the memoServer functionality
- */
+*/
 type ResolverDBStatic struct {
 	signatures map[string]Sig
 }
@@ -40,6 +40,9 @@ func (rds *ResolverDBStatic) Resolve(signature string) (*MediaEntry, error) {
 	if err != nil {
 		return nil, emperror.Wrapf(err, "cannot parse url %s of signature %s", sig.Uri, signature)
 	}
+	if u == nil {
+		return nil, emperror.Wrapf(err, "cannot parse url %s of signature %s", sig.Uri, signature)
+	}
 	p, ok := MediaProtocolString[sig.Type]
 	if !ok {
 		return nil, errors.New(fmt.Sprintf("invalid protocol %s for signature %s", sig.Type, signature))
@@ -50,7 +53,7 @@ func (rds *ResolverDBStatic) Resolve(signature string) (*MediaEntry, error) {
 	}
 	return &MediaEntry{
 		Signature: signature,
-		URI:       u,
+		URI:       *u,
 		Protocol:  p,
 		Access:    a,
 	}, nil