Verified Commit 36960de0 authored by Sebastian Schüpbach's avatar Sebastian Schüpbach
Browse files

cache thumbnails

parent ac647ded
Pipeline #42059 passed with stages
in 2 minutes and 42 seconds
......@@ -30,6 +30,7 @@ type CfgResolverDBMySQL struct {
type IIIFConfig struct {
Prefix string
Base string
Cached string
Url string
JwtSubPrefix string
ViewerTemplate string
......
......@@ -138,6 +138,7 @@ func main() {
config.CmdPrefix,
config.IIIF.Prefix,
config.IIIF.Base,
config.IIIF.Cached,
config.IIIF.Url,
config.IIIF.JwtSubPrefix,
config.IIIF.ViewerTemplate,
......
......@@ -55,9 +55,14 @@ folder = "C:/temp"
alias = "demo"
folder = "C:/daten/go/dev/streaming-server/web/static"
[[filemap]]
alias = "demo-cached"
folder = "C:/daten/go/dev/streaming-server/web/static/cached"
[iiif]
prefix = "/iiif/"
base = "C:/daten/go/dev/streaming-server/web/static/"
cached = "C:/daten/go/dev/streaming-server/web/static/cached"
url = "http://localhost:8182/iiif/2/"
jwtsubprefix = "iiif:"
viewertemplate = "C:/daten/go/dev/streaming-server/web/template/openseadragon.gohtml"
......
......@@ -38,9 +38,14 @@ folder = "C:/temp"
alias = "demo"
folder = "C:/daten/go/dev/streaming-server/web/static"
[[filemap]]
alias = "demo-cached"
folder = "C:/daten/go/dev/streaming-server/web/static/cached"
[iiif]
prefix = "/iiif/"
base = "C:/daten/go/dev/streaming-server/web/static/"
cached = "C:/daten/go/dev/streaming-server/web/static/cached"
url = "http://localhost:8182/iiif/2/"
jwtsubprefix = "iiif:"
viewertemplate = "C:/daten/go/dev/streaming-server/web/template/openseadragon.gohtml"
......
......@@ -30,9 +30,14 @@ baseurl = "https://media.memobase.k8s.unibas.ch/"
alias = "testdata"
folder = "/data/"
[[filemap]]
alias = "cached"
folder = "/data/cached/"
[iiif]
prefix = "/iiif/"
base = "/data/"
cached = "/data/cached/"
url = "http://api-image-server-prod-service:8182/iiif/2/"
jwtsubprefix = "iiif:"
viewertemplate = "/app/templates/openseadragon.gohtml"
......
......@@ -30,9 +30,14 @@ baseurl = "https://media-stage.memobase.k8s.unibas.ch/"
alias = "testdata"
folder = "/data/"
[[filemap]]
alias = "cached"
folder = "/data/cached/"
[iiif]
prefix = "/iiif/"
base = "/data/"
cached = "/data/cached/"
url = "http://api-image-server-stage-service:8182/iiif/2/"
jwtsubprefix = "iiif:"
viewertemplate = "/app/templates/openseadragon.gohtml"
......
......@@ -30,9 +30,14 @@ baseurl = "https://media-test.memobase.k8s.unibas.ch/"
alias = "testdata"
folder = "/data/"
[[filemap]]
alias = "cached"
folder = "/data/cached/"
[iiif]
prefix = "/iiif/"
base = "/data/"
cached = "/data/cached/"
url = "http://api-image-server-test-service:8182/iiif/2/"
jwtsubprefix = "iiif:"
viewertemplate = "/app/templates/openseadragon.gohtml"
......
package memostream
import (
"errors"
"fmt"
"github.com/goph/emperror"
"io/ioutil"
"net/http"
"os"
"path/filepath"
"regexp"
"strconv"
......@@ -61,7 +64,7 @@ func (iai *ActionImageIIIF) iiif(w http.ResponseWriter, req *http.Request, me *M
return false, fmt.Errorf("invalid parameter %s", p)
}
}
if err := iai.ms.proxyIIIF(req, w, me.Signature, file, strings.Join(params, "/"), me.Access == Media_Public); err != nil {
if _, err := iai.ms.proxyIIIF(req, w, me.Signature, file, strings.Join(params, "/"), me.Access == Media_Public); err != nil {
return false, fmt.Errorf("cannot proxy to iiif server: %v", err)
}
return true, nil
......@@ -117,6 +120,9 @@ func (iai *ActionImageIIIF) resize(w http.ResponseWriter, req *http.Request, me
}
}
// A thumbnail is an otherwise unaltered representation of an image with size 640x480
wantsThumbnail := true
// calculate width and height from resize parameter
var width, height int64
var err error
......@@ -134,16 +140,22 @@ func (iai *ActionImageIIIF) resize(w http.ResponseWriter, req *http.Request, me
return false, emperror.Wrapf(err, "invalid height %v in resize", resizeParams[1])
}
if width != 640 || height != 480 {
wantsThumbnail = false
}
size := fmt.Sprintf("%v,%v", width, height)
var format string
if formatParams, ok := ps["format"]; ok {
format = strings.ToLower(formatParams[0])
wantsThumbnail = false
}
var ext string
switch format {
case "png":
ext = "png"
wantsThumbnail = false
default:
ext = "jpg"
}
......@@ -151,15 +163,19 @@ func (iai *ActionImageIIIF) resize(w http.ResponseWriter, req *http.Request, me
// if not stretch, we need !size
if _, ok := ps["stretch"]; !ok {
size = "!" + size
} else {
wantsThumbnail = false
}
rotate := "0"
if rot, ok := ps["rotate"]; ok {
rotate = rot[0]
wantsThumbnail = false
}
if _, ok := ps["flip"]; ok {
rotate = "!" + rotate
wantsThumbnail = false
}
color := "default"
......@@ -170,6 +186,7 @@ func (iai *ActionImageIIIF) resize(w http.ResponseWriter, req *http.Request, me
if color == "" {
color = "color"
}
wantsThumbnail = false
}
resultName := fmt.Sprintf("%s_%vx%v.%s", me.Signature, width, height, ext)
......@@ -198,6 +215,7 @@ func (iai *ActionImageIIIF) resize(w http.ResponseWriter, req *http.Request, me
nY = (me.Height - nH) / 2
}
region = fmt.Sprintf("%v,%v,%v,%v", nX, nY, nW, nH)
wantsThumbnail = false
}
// todo: region & rotation parameter
......@@ -217,9 +235,72 @@ func (iai *ActionImageIIIF) resize(w http.ResponseWriter, req *http.Request, me
sigfile = strings.TrimPrefix(sigfile, iai.ms.iiifBase)
sigfile = strings.Replace(sigfile, "/", "%24", -1)
if err := iai.ms.proxyIIIF(req, w, me.Signature, sigfile, paramstring, me.Access == Media_Public); err != nil {
if wantsThumbnail {
if ok, err := iai.tryFetchCachedThumbnail(w, me); ok && err == nil {
return true, nil
} else if err != nil {
iai.ms.log.Error("Cannot fetch cached thumbnail")
return false, fmt.Errorf("cannot fetch cached thumbnail: %v", err)
}
}
iai.ms.log.Info("Fetching file from image server")
if buf, err := iai.ms.proxyIIIF(req, w, me.Signature, sigfile, paramstring, me.Access == Media_Public); err != nil {
return false, fmt.Errorf("cannot proxy request: %v", err)
} else if wantsThumbnail {
iai.cacheThumbnail(buf, me)
}
return true, nil
}
func (iai *ActionImageIIIF) tryFetchCachedThumbnail(w http.ResponseWriter, me *MediaEntry) (bool, error) {
cachedUri := iai.generatePathToCachedImage(me)
if _, err := os.Stat(cachedUri); errors.Is(err, os.ErrNotExist) {
iai.ms.log.Debugf("Thumbnail %v is not yet cached", cachedUri)
return false, nil
}
fileBytes, err := ioutil.ReadFile(cachedUri)
if err != nil {
iai.ms.log.Errorf("File read error %v", cachedUri)
return false, err
}
w.WriteHeader(http.StatusOK)
w.Header().Set("Content-Type", me.Mimetype)
w.Write(fileBytes)
iai.ms.log.Debugf("Thumbnail for %v already in cache", cachedUri)
return true, nil
}
func (iai *ActionImageIIIF) cacheThumbnail(imageBuf []byte, me *MediaEntry) (bool, error) {
cachedUri := iai.generatePathToCachedImage(me)
iai.ms.log.Debugf("Length of %v: %v", cachedUri, len(imageBuf))
f, err := os.Create(cachedUri)
if err != nil {
iai.ms.log.Errorf("An error occurred when creating cache file: %v", err)
return false, err
}
if _, err := f.Write(imageBuf); err != nil {
iai.ms.log.Errorf("An error occurred when writing cache file: %v", err)
return false, err
}
if err := f.Close(); err != nil {
iai.ms.log.Errorf("An error occurred when closing cache file: %v", err)
return false, err
}
return true, nil
}
func (iai *ActionImageIIIF) generatePathToCachedImage(me *MediaEntry) string {
if me.URI.Scheme == "http" || me.URI.Scheme == "https" {
var mimetype string
if me.Mimetype == "image/jpeg" || me.Mimetype == "image/jpg" {
mimetype = "jpg"
} else {
mimetype = "png"
}
return fmt.Sprintf("%v%v.%v", iai.ms.iiifCached, me.Signature, mimetype)
} else if me.URI.Scheme == "file" {
return strings.Replace(me.URI.String()[7:], iai.ms.iiifBase, iai.ms.iiifCached, 1)
}
return me.URI.String()
}
......@@ -13,7 +13,7 @@ type ReadSeekerCloser interface {
io.Seeker
}
// abstraction interface for creating ReasSeekerCloser
// abstraction interface for creating ReadSeekerCloser
type Filesystem interface {
// creates ReadSeekerCloser, returns last modification time
Open(uri url.URL) (ReadSeekerCloser, time.Time, error)
......
......@@ -41,6 +41,7 @@ type memoServer struct {
staticPrefix string
iiifPrefix string
iiifBase string
iiifCached string
iiifUrl string
iiifJwtSubPrefix string
iiifManifestBase string
......@@ -64,6 +65,7 @@ func NewServer(
cmdPrefix,
iiifPrefix,
iiifBase,
iiifCached,
iiifUrl,
iiifJwtSubPrefix,
iiifViewerTemplate,
......@@ -110,6 +112,7 @@ func NewServer(
cmdPrefix: cmdPrefix,
iiifPrefix: iiifPrefix,
iiifBase: iiifBase,
iiifCached: iiifCached,
iiifUrl: iiifUrl,
iiifJwtSubPrefix: iiifJwtSubPrefix,
iiifManifestBase: iiifManifestBase,
......
......@@ -11,16 +11,18 @@
package memostream
import (
"bytes"
"github.com/goph/emperror"
"github.com/gorilla/mux"
"io"
"io/ioutil"
"net/http"
"net/url"
"path/filepath"
"strings"
)
func (ms *memoServer) proxyIIIF(req *http.Request, writer http.ResponseWriter, signature, file, params string, public bool) error {
func (ms *memoServer) proxyIIIF(req *http.Request, writer http.ResponseWriter, signature, file, params string, public bool) ([]byte, error) {
// create path as url part valid for mediaserver
//file := strings.Replace(_file, "$", "%24", -1)
file = strings.TrimPrefix(file, ms.iiifBase)
......@@ -35,7 +37,7 @@ func (ms *memoServer) proxyIIIF(req *http.Request, writer http.ResponseWriter, s
ms.log.Infof("Proxy: %v", urlstring)
req2, err := http.NewRequest("GET", urlstring, nil)
if err != nil {
return emperror.Wrapf(err, "Error creating new request")
return nil, emperror.Wrapf(err, "Error creating new request")
//ms.DoPanicf(writer, http.StatusInternalServerError, "Error creating new request: %v", err)
//return
}
......@@ -46,14 +48,14 @@ func (ms *memoServer) proxyIIIF(req *http.Request, writer http.ResponseWriter, s
// NewJWT(secret string, subject string, alg string, valid int64, domain string, issuer string)
newtoken, err = NewJWT(ms.jwtSecret, sub, ms.jwtAlg[0], 7200, "", "")
if err != nil {
return emperror.Wrapf(err, "Error creating access token")
return nil, emperror.Wrapf(err, "Error creating access token")
}
}
// build headers to send cantaloupe the real url's to use
//proto, host, port := ms.getProtoHostPort(req)
baseurl, err := url.Parse(ms.baseUrl)
if err != nil {
return emperror.Wrapf(err, "Cannot parse baseurl: %s", ms.baseUrl)
return nil, emperror.Wrapf(err, "Cannot parse baseurl: %s", ms.baseUrl)
}
req2.Header.Add("X-Forwarded-Proto", baseurl.Scheme)
req2.Header.Add("X-Forwarded-Host", baseurl.Hostname())
......@@ -63,14 +65,18 @@ func (ms *memoServer) proxyIIIF(req *http.Request, writer http.ResponseWriter, s
rs, err := client.Do(req2)
if err != nil {
return emperror.Wrapf(err, "Error calling proxy: %s", urlstring)
return nil, emperror.Wrapf(err, "Error calling proxy: %s", urlstring)
}
defer rs.Body.Close()
buf, err := ioutil.ReadAll(rs.Body)
rs.Body = ioutil.NopCloser(bytes.NewBuffer(buf))
io.Copy(writer, rs.Body)
return nil
if err != nil {
ms.log.Error(err)
}
return buf, nil
}
// IIIF handler
......@@ -142,7 +148,7 @@ func (ms *memoServer) HandlerIIIF(writer http.ResponseWriter, req *http.Request)
return
}
if err := ms.proxyIIIF(req, writer, signature, strings.Replace(file, "$", "%24", -1), params, me.Access == Media_Public); err != nil {
if _, err := ms.proxyIIIF(req, writer, signature, strings.Replace(file, "$", "%24", -1), params, me.Access == Media_Public); err != nil {
ms.DoPanicf(writer, http.StatusInternalServerError, "cannot proxy request: %v", err)
return
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment