Commit dbea6c82 authored by Jürgen Enge's avatar Jürgen Enge
Browse files

webservice active

parent 9d38915e
...@@ -19,25 +19,47 @@ func (d *duration) UnmarshalText(text []byte) error { ...@@ -19,25 +19,47 @@ func (d *duration) UnmarshalText(text []byte) error {
type ConfigImageMagick struct { type ConfigImageMagick struct {
Identify string Identify string
Convert string Convert string
Wsl bool }
type ConfigFFMPEG struct {
FFMPEG string
FFPROBE string
}
type ConfigHistogram struct {
Prefix string
Colormap map[string]string
Timeout duration Timeout duration
Remap string Remap string
Colors int Colors int
Resize string Resize string
} }
type ConfigValidateImage struct {
Prefix string
Timeout duration
}
type ConfigValidateAV struct {
Prefix string
Timeout duration
}
type Config struct { type Config struct {
Logfile string Logfile string
Loglevel string Loglevel string
AccessLog string AccessLog string
CertPEM string CertPEM string
KeyPEM string KeyPEM string
Addr string Addr string
HistogramPrefix string JwtKey string
JwtKey string JwtAlg []string
JwtAlg []string ImageMagick ConfigImageMagick
ImageMagick ConfigImageMagick ConfigFFMPEG ConfigFFMPEG
Colormap map[string]string Histogram ConfigHistogram
ValidateImage ConfigValidateImage
ValidateAV ConfigValidateAV
Wsl bool
} }
func LoadConfig(filepath string) Config { func LoadConfig(filepath string) Config {
......
...@@ -6,7 +6,6 @@ import ( ...@@ -6,7 +6,6 @@ import (
"fmt" "fmt"
"gitlab.switch.ch/memoriav/memobase-2020/services/histogram/pkg/histogram" "gitlab.switch.ch/memoriav/memobase-2020/services/histogram/pkg/histogram"
"gitlab.switch.ch/memoriav/memobase-2020/services/histogram/pkg/service" "gitlab.switch.ch/memoriav/memobase-2020/services/histogram/pkg/service"
"io"
"log" "log"
"os" "os"
"path/filepath" "path/filepath"
...@@ -40,28 +39,15 @@ func main() { ...@@ -40,28 +39,15 @@ func main() {
log, lf := service.CreateLogger("indexer", config.Logfile, config.Loglevel) log, lf := service.CreateLogger("indexer", config.Logfile, config.Loglevel)
defer lf.Close() defer lf.Close()
var accesslog io.Writer
if config.AccessLog == "" {
accesslog = os.Stdout
} else {
f, err := os.OpenFile(config.AccessLog, os.O_WRONLY|os.O_CREATE, 0755)
if err != nil {
log.Panicf("cannot open file %s: %v", config.AccessLog, err)
return
}
defer f.Close()
accesslog = f
}
fmt.Sprintf("%v", accesslog)
hist, err := histogram.NewHistogram( hist, err := histogram.NewHistogram(
config.ImageMagick.Convert, config.ImageMagick.Convert,
config.ImageMagick.Resize, config.Histogram.Resize,
config.ImageMagick.Remap, config.Histogram.Remap,
config.Colormap, config.Histogram.Colormap,
config.ImageMagick.Colors, config.Histogram.Colors,
config.ImageMagick.Timeout.Duration, config.Histogram.Timeout.Duration,
config.ImageMagick.Wsl) config.Wsl)
// if necessary create a colormap image // if necessary create a colormap image
result, err := hist.Exec(*imgFile) result, err := hist.Exec(*imgFile)
......
package main
import (
"github.com/BurntSushi/toml"
"log"
"time"
)
type duration struct {
Duration time.Duration
}
func (d *duration) UnmarshalText(text []byte) error {
var err error
d.Duration, err = time.ParseDuration(string(text))
return err
}
type ConfigImageMagick struct {
Identify string
Convert string
}
type ConfigFFMPEG struct {
FFMPEG string
FFPROBE string
}
type ConfigHistogram struct {
Prefix string
Colormap map[string]string
Timeout duration
Remap string
Colors int
Resize string
}
type ConfigValidateImage struct {
Prefix string
Timeout duration
}
type ConfigValidateAV struct {
Prefix string
Timeout duration
}
type Config struct {
Logfile string
Loglevel string
AccessLog string
CertPEM string
KeyPEM string
Addr string
JwtKey string
JwtAlg []string
ImageMagick ConfigImageMagick
FFMPEG ConfigFFMPEG
Histogram ConfigHistogram
ValidateImage ConfigValidateImage
ValidateAV ConfigValidateAV
Wsl bool
}
func LoadConfig(filepath string) Config {
var conf Config
_, err := toml.DecodeFile(filepath, &conf)
if err != nil {
log.Fatalln("Error on loading config: ", err)
}
return conf
}
package main
import (
"context"
"flag"
"fmt"
"gitlab.switch.ch/memoriav/memobase-2020/services/histogram/pkg/histogram"
"gitlab.switch.ch/memoriav/memobase-2020/services/histogram/pkg/service"
"gitlab.switch.ch/memoriav/memobase-2020/services/histogram/pkg/validate"
"io"
"log"
"os"
"os/signal"
"path/filepath"
"syscall"
"time"
)
func main() {
configFile := flag.String("cfg", "./histogram.toml", "config file location")
flag.Parse()
var exPath = ""
// if configfile not found try path of executable as prefix
if !service.FileExists(*configFile) {
ex, err := os.Executable()
if err != nil {
panic(err)
}
exPath = filepath.Dir(ex)
if service.FileExists(filepath.Join(exPath, *configFile)) {
*configFile = filepath.Join(exPath, *configFile)
} else {
log.Fatalf("cannot find configuration file: %v", *configFile)
return
}
}
// configfile should exists at this place
var config Config
config = LoadConfig(*configFile)
// create logger instance
log, lf := service.CreateLogger("indexer", config.Logfile, config.Loglevel)
defer lf.Close()
var accesslog io.Writer
if config.AccessLog == "" {
accesslog = os.Stdout
} else {
f, err := os.OpenFile(config.AccessLog, os.O_WRONLY|os.O_CREATE, 0755)
if err != nil {
log.Panicf("cannot open file %s: %v", config.AccessLog, err)
return
}
defer f.Close()
accesslog = f
}
fmt.Sprintf("%v", accesslog)
imgval, err := validate.NewValidateImage(config.ImageMagick.Identify, config.ValidateImage.Timeout.Duration, config.Wsl, log)
if err != nil {
log.Fatalf("cannot initialize ValidateImage: %v", err)
return
}
avval, err := validate.NewValidateAV(config.FFMPEG.FFMPEG, config.ValidateAV.Timeout.Duration, config.Wsl, log)
if err != nil {
log.Fatalf("cannot initialize ValidateAV: %v", err)
return
}
hist, err := histogram.NewHistogram(
config.ImageMagick.Convert,
config.Histogram.Resize,
config.Histogram.Remap,
config.Histogram.Colormap,
config.Histogram.Colors,
config.Histogram.Timeout.Duration,
config.Wsl)
if err != nil {
log.Fatalf("cannot initialize histogram: %v", err)
return
}
srv, err := service.NewServer(
config.Addr,
log,
accesslog,
map[string]service.Service{
config.Histogram.Prefix: hist,
config.ValidateImage.Prefix: imgval,
config.ValidateAV.Prefix: avval,
},
config.Wsl,
)
if err != nil {
log.Fatalf("cannot initialize server: %v", err)
return
}
go func() {
if err := srv.ListenAndServe(config.CertPEM, config.KeyPEM); err != nil {
log.Errorf("server died: %v", err)
}
}()
end := make(chan bool, 1)
// process waiting for interrupt signal (TERM or KILL)
go func() {
sigint := make(chan os.Signal, 1)
// interrupt signal sent from terminal
signal.Notify(sigint, os.Interrupt)
signal.Notify(sigint, syscall.SIGTERM)
signal.Notify(sigint, syscall.SIGKILL)
<-sigint
// We received an interrupt signal, shut down.
log.Infof("shutdown requested")
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
srv.Shutdown(ctx)
end <- true
}()
<-end
log.Info("server stopped")
}
...@@ -4,12 +4,26 @@ accesslog = "" # http access log file ...@@ -4,12 +4,26 @@ accesslog = "" # http access log file
addr = "localhost:83" addr = "localhost:83"
#certpem = "" # tls client certificate file in PEM format #certpem = "" # tls client certificate file in PEM format
#keypem = "" # tls client key file in PEM format #keypem = "" # tls client key file in PEM format
histogramprefix = "histogram" wsl = true # true, if executable is within linux subsystem on windows
[ImageMagick] [ImageMagick]
#identify = "/usr/bin/identify" identify = "/usr/bin/identify"
convert = "/usr/bin/convert" convert = "/usr/bin/convert"
wsl = true # true, if executable is within linux subsystem on windows
[FFMPEG]
ffmpeg = "/usr/local/bin/ffmpeg2"
ffprobe = "/usr/local/bin/ffprobe2"
[ValidateImage]
prefix = "validateimage"
timeout = "10s"
[ValidateAV]
prefix = "validateav"
timeout = "10s"
[Histogram]
prefix = "histogram"
timeout = "10s" timeout = "10s"
remap = "color.png" # das Bild mit der Zielfarbpalette. Wird erzeugt, falls nicht vorhanden. (Schreibrechte beachten) remap = "color.png" # das Bild mit der Zielfarbpalette. Wird erzeugt, falls nicht vorhanden. (Schreibrechte beachten)
colors = 10 # Anzahl der Farben im Histogramm colors = 10 # Anzahl der Farben im Histogramm
......
Image: parliamentdefect.jpg
Format: JPEG (Joint Photographic Experts Group JFIF format)
Mime type: image/jpeg
Class: DirectClass
Geometry: 2480x1394+0+0
Resolution: 150x150
Print size: 16.5333x9.29333
Units: PixelsPerInch
Type: TrueColor
Endianess: Undefined
Colorspace: sRGB
Depth: 8-bit
Channel depth:
red: 8-bit
green: 8-bit
blue: 8-bit
Channel statistics:
Pixels: 3457120
Red:
min: 0 (0)
max: 255 (1)
mean: 99.1014 (0.388633)
standard deviation: 62.826 (0.246376)
kurtosis: -0.177222
skewness: 1.00838
entropy: 0.922141
Green:
min: 0 (0)
max: 255 (1)
mean: 84.6732 (0.332052)
standard deviation: 58.3896 (0.228979)
kurtosis: 0.773212
skewness: 1.30255
entropy: 0.893161
Blue:
min: 0 (0)
max: 255 (1)
mean: 78.7253 (0.308727)
standard deviation: 56.2547 (0.220607)
kurtosis: 1.76577
skewness: 1.60893
entropy: 0.882406
Image statistics:
Overall:
min: 0 (0)
max: 255 (1)
mean: 87.5 (0.343137)
standard deviation: 59.2201 (0.232235)
kurtosis: 0.767738
skewness: 1.319
entropy: 0.899236
Rendering intent: Perceptual
Gamma: 0.454545
Chromaticity:
red primary: (0.64,0.33)
green primary: (0.3,0.6)
blue primary: (0.15,0.06)
white point: (0.3127,0.329)
Background color: white
Border color: srgb(223,223,223)
Matte color: grey74
Transparent color: black
Interlace: None
Intensity: Undefined
Compose: Over
Page geometry: 2480x1394+0+0
Dispose: Undefined
Iterations: 0
Compression: JPEG
Quality: 90
Orientation: Undefined
Properties:
date:create: 2020-06-18T12:11:31+02:00
date:modify: 2020-06-18T12:11:31+02:00
jpeg:colorspace: 2
jpeg:sampling-factor: 2x2,1x1,1x1
signature: 0c81f2574d32dda8581b104c2f33c751ab2b9e43ad2335dcc195acd95df73d7b
Artifacts:
filename: parliamentdefect.jpg
verbose: true
Tainted: False
Filesize: 988KB
Number pixels: 3.457M
Pixels per second: 69.14MB
User time: 0.040u
Elapsed time: 0:01.049
Version: ImageMagick 6.9.7-4 Q16 x86_64 20170114 http://www.imagemagick.org
...@@ -60,9 +60,7 @@ func NewHistogram(convert, resize, remap string, colormap map[string]string, col ...@@ -60,9 +60,7 @@ func NewHistogram(convert, resize, remap string, colormap map[string]string, col
return h, nil return h, nil
} }
func (h *Histogram) GetName() string {
return "histogram"
}
func (h *Histogram) Exec(file string, args ...interface{}) (interface{}, error) { func (h *Histogram) Exec(file string, args ...interface{}) (interface{}, error) {
colors := make(map[string]int64) colors := make(map[string]int64)
......
...@@ -11,6 +11,9 @@ import ( ...@@ -11,6 +11,9 @@ import (
"io" "io"
"net" "net"
"net/http" "net/http"
"net/url"
"regexp"
"runtime"
) )
type Server struct { type Server struct {
...@@ -20,6 +23,7 @@ type Server struct { ...@@ -20,6 +23,7 @@ type Server struct {
log *logging.Logger log *logging.Logger
accesslog io.Writer accesslog io.Writer
services map[string]Service services map[string]Service
wsl bool
} }
func NewServer( func NewServer(
...@@ -27,6 +31,7 @@ func NewServer( ...@@ -27,6 +31,7 @@ func NewServer(
log *logging.Logger, log *logging.Logger,
accesslog io.Writer, accesslog io.Writer,
srvs map[string]Service, srvs map[string]Service,
wsl bool,
) (*Server, error) { ) (*Server, error) {
host, port, err := net.SplitHostPort(addr) host, port, err := net.SplitHostPort(addr)
if err != nil { if err != nil {
...@@ -41,6 +46,7 @@ func NewServer( ...@@ -41,6 +46,7 @@ func NewServer(
log: log, log: log,
accesslog: accesslog, accesslog: accesslog,
services: srvs, services: srvs,
wsl: wsl,
} }
return srv, nil return srv, nil
} }
...@@ -48,24 +54,58 @@ func NewServer( ...@@ -48,24 +54,58 @@ func NewServer(
func (s *Server) ListenAndServe(cert, key string) error { func (s *Server) ListenAndServe(cert, key string) error {
router := mux.NewRouter() router := mux.NewRouter()
for prefix, serv := range s.services { srvRegexp := regexp.MustCompile(`^/([^/]+)/(.+)$`)
router.MatcherFunc(func(r *http.Request, rm *mux.RouteMatch) bool {
router.HandleFunc(fmt.Sprintf("/%s/[path]", prefix), func(writer http.ResponseWriter, request *http.Request) { matches := srvRegexp.FindSubmatch([]byte(r.URL.Path))
vars := mux.Vars(request) if len(matches) == 0 {
path, ok := vars["path"] return false
if !ok { }
s.DoPanicf(writer, http.StatusNotFound, "invalid url %s. no path given", request.URL.String()) rm.Vars = map[string]string{}
return prefix := string(matches[1])
} rm.Vars["prefix"] = prefix
result, err := serv.Exec(path) rm.Vars["path"] = string(matches[2])
if err != nil { // check whether prefix is known
s.DoPanicf(writer, http.StatusInternalServerError, "cannot create histogram of %s", path) for key, _ := range s.services {
return if key == prefix {
return true
} }
jw := json.NewEncoder(writer) }
jw.Encode(result) return false
}).Methods("GET") }).HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
} writer.Header().Set("Content-Type", "application/json")
vars := mux.Vars(request)
prefix, ok := vars["prefix"]
if !ok {
s.DoPanicf(writer, http.StatusNotFound, "invalid url %s. no path given", request.URL.String())
return
}
serv, ok := s.services[prefix]
if !ok {
s.DoPanicf(writer, http.StatusNotFound, "invalid url %s. wrong prefix %s", request.URL.String(), prefix)
return
}
path, ok := vars["path"]
if !ok {
s.DoPanicf(writer, http.StatusNotFound, "invalid url %s. no path given", request.URL.String())
return
}
if runtime.GOOS != "windows" || (runtime.GOOS == "windows" && s.wsl) {
path = `/` + path
}
p, err := url.QueryUnescape(path)
if err != nil {
s.DoPanicf(writer, http.StatusInternalServerError, "cannot unescape path %s: %v", path, err)
return
}
result, err := serv.Exec(p)
if err != nil {
s.DoPanicf(writer, http.StatusInternalServerError, "cannot create %s of %s: %v", prefix, path, err)
return
}
jw := json.NewEncoder(writer)
jw.Encode(result)
}).Methods("GET")
loggedRouter := handlers.LoggingHandler(s.accesslog, router) loggedRouter := handlers.LoggingHandler(s.accesslog, router)
addr := net.JoinHostPort(s.host, s.port) addr := net.JoinHostPort(s.host, s.port)
......
...@@ -2,5 +2,4 @@ package service ...@@ -2,5 +2,4 @@ package service
type Service interface { type Service interface {
Exec(filename string, args... interface{}) (interface{}, error); Exec(filename string, args... interface{}) (interface{}, error);
GetName() string;
} }
package validate
import (
"bytes"
"context"
"github.com/goph/emperror"
"github.com/op/go-logging"
"os/exec"
"strings"
"time"
)
type ValidateAV struct {
log *logging.Logger
ffmpeg string