Commit dbea6c82 authored by Jürgen Enge's avatar Jürgen Enge
Browse files

webservice active

parent 9d38915e
......@@ -19,25 +19,47 @@ func (d *duration) UnmarshalText(text []byte) error {
type ConfigImageMagick struct {
Identify string
Convert string
Wsl bool
}
type ConfigFFMPEG struct {
FFMPEG string
FFPROBE string
}
type ConfigHistogram struct {
Prefix string
Colormap map[string]string
Timeout duration
Remap string
Colors int
Resize string
}
type ConfigValidateImage struct {
Prefix string
Timeout duration
}
type ConfigValidateAV struct {
Prefix string
Timeout duration
}
type Config struct {
Logfile string
Loglevel string
AccessLog string
CertPEM string
KeyPEM string
Addr string
HistogramPrefix string
JwtKey string
JwtAlg []string
ImageMagick ConfigImageMagick
Colormap map[string]string
Logfile string
Loglevel string
AccessLog string
CertPEM string
KeyPEM string
Addr string
JwtKey string
JwtAlg []string
ImageMagick ConfigImageMagick
ConfigFFMPEG ConfigFFMPEG
Histogram ConfigHistogram
ValidateImage ConfigValidateImage
ValidateAV ConfigValidateAV
Wsl bool
}
func LoadConfig(filepath string) Config {
......
......@@ -6,7 +6,6 @@ import (
"fmt"
"gitlab.switch.ch/memoriav/memobase-2020/services/histogram/pkg/histogram"
"gitlab.switch.ch/memoriav/memobase-2020/services/histogram/pkg/service"
"io"
"log"
"os"
"path/filepath"
......@@ -40,28 +39,15 @@ func main() {
log, lf := service.CreateLogger("indexer", config.Logfile, config.Loglevel)
defer lf.Close()
var accesslog io.Writer
if config.AccessLog == "" {
accesslog = os.Stdout
} else {
f, err := os.OpenFile(config.AccessLog, os.O_WRONLY|os.O_CREATE, 0755)
if err != nil {
log.Panicf("cannot open file %s: %v", config.AccessLog, err)
return
}
defer f.Close()
accesslog = f
}
fmt.Sprintf("%v", accesslog)
hist, err := histogram.NewHistogram(
config.ImageMagick.Convert,
config.ImageMagick.Resize,
config.ImageMagick.Remap,
config.Colormap,
config.ImageMagick.Colors,
config.ImageMagick.Timeout.Duration,
config.ImageMagick.Wsl)
config.Histogram.Resize,
config.Histogram.Remap,
config.Histogram.Colormap,
config.Histogram.Colors,
config.Histogram.Timeout.Duration,
config.Wsl)
// if necessary create a colormap image
result, err := hist.Exec(*imgFile)
......
package main
import (
"github.com/BurntSushi/toml"
"log"
"time"
)
type duration struct {
Duration time.Duration
}
func (d *duration) UnmarshalText(text []byte) error {
var err error
d.Duration, err = time.ParseDuration(string(text))
return err
}
type ConfigImageMagick struct {
Identify string
Convert string
}
type ConfigFFMPEG struct {
FFMPEG string
FFPROBE string
}
type ConfigHistogram struct {
Prefix string
Colormap map[string]string
Timeout duration
Remap string
Colors int
Resize string
}
type ConfigValidateImage struct {
Prefix string
Timeout duration
}
type ConfigValidateAV struct {
Prefix string
Timeout duration
}
type Config struct {
Logfile string
Loglevel string
AccessLog string
CertPEM string
KeyPEM string
Addr string
JwtKey string
JwtAlg []string
ImageMagick ConfigImageMagick
FFMPEG ConfigFFMPEG
Histogram ConfigHistogram
ValidateImage ConfigValidateImage
ValidateAV ConfigValidateAV
Wsl bool
}
func LoadConfig(filepath string) Config {
var conf Config
_, err := toml.DecodeFile(filepath, &conf)
if err != nil {
log.Fatalln("Error on loading config: ", err)
}
return conf
}
package main
import (
"context"
"flag"
"fmt"
"gitlab.switch.ch/memoriav/memobase-2020/services/histogram/pkg/histogram"
"gitlab.switch.ch/memoriav/memobase-2020/services/histogram/pkg/service"
"gitlab.switch.ch/memoriav/memobase-2020/services/histogram/pkg/validate"
"io"
"log"
"os"
"os/signal"
"path/filepath"
"syscall"
"time"
)
func main() {
configFile := flag.String("cfg", "./histogram.toml", "config file location")
flag.Parse()
var exPath = ""
// if configfile not found try path of executable as prefix
if !service.FileExists(*configFile) {
ex, err := os.Executable()
if err != nil {
panic(err)
}
exPath = filepath.Dir(ex)
if service.FileExists(filepath.Join(exPath, *configFile)) {
*configFile = filepath.Join(exPath, *configFile)
} else {
log.Fatalf("cannot find configuration file: %v", *configFile)
return
}
}
// configfile should exists at this place
var config Config
config = LoadConfig(*configFile)
// create logger instance
log, lf := service.CreateLogger("indexer", config.Logfile, config.Loglevel)
defer lf.Close()
var accesslog io.Writer
if config.AccessLog == "" {
accesslog = os.Stdout
} else {
f, err := os.OpenFile(config.AccessLog, os.O_WRONLY|os.O_CREATE, 0755)
if err != nil {
log.Panicf("cannot open file %s: %v", config.AccessLog, err)
return
}
defer f.Close()
accesslog = f
}
fmt.Sprintf("%v", accesslog)
imgval, err := validate.NewValidateImage(config.ImageMagick.Identify, config.ValidateImage.Timeout.Duration, config.Wsl, log)
if err != nil {
log.Fatalf("cannot initialize ValidateImage: %v", err)
return
}
avval, err := validate.NewValidateAV(config.FFMPEG.FFMPEG, config.ValidateAV.Timeout.Duration, config.Wsl, log)
if err != nil {
log.Fatalf("cannot initialize ValidateAV: %v", err)
return
}
hist, err := histogram.NewHistogram(
config.ImageMagick.Convert,
config.Histogram.Resize,
config.Histogram.Remap,
config.Histogram.Colormap,
config.Histogram.Colors,
config.Histogram.Timeout.Duration,
config.Wsl)
if err != nil {
log.Fatalf("cannot initialize histogram: %v", err)
return
}
srv, err := service.NewServer(
config.Addr,
log,
accesslog,
map[string]service.Service{
config.Histogram.Prefix: hist,
config.ValidateImage.Prefix: imgval,
config.ValidateAV.Prefix: avval,
},
config.Wsl,
)
if err != nil {
log.Fatalf("cannot initialize server: %v", err)
return
}
go func() {
if err := srv.ListenAndServe(config.CertPEM, config.KeyPEM); err != nil {
log.Errorf("server died: %v", err)
}
}()
end := make(chan bool, 1)
// process waiting for interrupt signal (TERM or KILL)
go func() {
sigint := make(chan os.Signal, 1)
// interrupt signal sent from terminal
signal.Notify(sigint, os.Interrupt)
signal.Notify(sigint, syscall.SIGTERM)
signal.Notify(sigint, syscall.SIGKILL)
<-sigint
// We received an interrupt signal, shut down.
log.Infof("shutdown requested")
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
srv.Shutdown(ctx)
end <- true
}()
<-end
log.Info("server stopped")
}
......@@ -4,12 +4,26 @@ accesslog = "" # http access log file
addr = "localhost:83"
#certpem = "" # tls client certificate file in PEM format
#keypem = "" # tls client key file in PEM format
histogramprefix = "histogram"
wsl = true # true, if executable is within linux subsystem on windows
[ImageMagick]
#identify = "/usr/bin/identify"
identify = "/usr/bin/identify"
convert = "/usr/bin/convert"
wsl = true # true, if executable is within linux subsystem on windows
[FFMPEG]
ffmpeg = "/usr/local/bin/ffmpeg2"
ffprobe = "/usr/local/bin/ffprobe2"
[ValidateImage]
prefix = "validateimage"
timeout = "10s"
[ValidateAV]
prefix = "validateav"
timeout = "10s"
[Histogram]
prefix = "histogram"
timeout = "10s"
remap = "color.png" # das Bild mit der Zielfarbpalette. Wird erzeugt, falls nicht vorhanden. (Schreibrechte beachten)
colors = 10 # Anzahl der Farben im Histogramm
......
Image: parliamentdefect.jpg
Format: JPEG (Joint Photographic Experts Group JFIF format)
Mime type: image/jpeg
Class: DirectClass
Geometry: 2480x1394+0+0
Resolution: 150x150
Print size: 16.5333x9.29333
Units: PixelsPerInch
Type: TrueColor
Endianess: Undefined
Colorspace: sRGB
Depth: 8-bit
Channel depth:
red: 8-bit
green: 8-bit
blue: 8-bit
Channel statistics:
Pixels: 3457120
Red:
min: 0 (0)
max: 255 (1)
mean: 99.1014 (0.388633)
standard deviation: 62.826 (0.246376)
kurtosis: -0.177222
skewness: 1.00838
entropy: 0.922141
Green:
min: 0 (0)
max: 255 (1)
mean: 84.6732 (0.332052)
standard deviation: 58.3896 (0.228979)
kurtosis: 0.773212
skewness: 1.30255
entropy: 0.893161
Blue:
min: 0 (0)
max: 255 (1)
mean: 78.7253 (0.308727)
standard deviation: 56.2547 (0.220607)
kurtosis: 1.76577
skewness: 1.60893
entropy: 0.882406
Image statistics:
Overall:
min: 0 (0)
max: 255 (1)
mean: 87.5 (0.343137)
standard deviation: 59.2201 (0.232235)
kurtosis: 0.767738
skewness: 1.319
entropy: 0.899236
Rendering intent: Perceptual
Gamma: 0.454545
Chromaticity:
red primary: (0.64,0.33)
green primary: (0.3,0.6)
blue primary: (0.15,0.06)
white point: (0.3127,0.329)
Background color: white
Border color: srgb(223,223,223)
Matte color: grey74
Transparent color: black
Interlace: None
Intensity: Undefined
Compose: Over
Page geometry: 2480x1394+0+0
Dispose: Undefined
Iterations: 0
Compression: JPEG
Quality: 90
Orientation: Undefined
Properties:
date:create: 2020-06-18T12:11:31+02:00
date:modify: 2020-06-18T12:11:31+02:00
jpeg:colorspace: 2
jpeg:sampling-factor: 2x2,1x1,1x1
signature: 0c81f2574d32dda8581b104c2f33c751ab2b9e43ad2335dcc195acd95df73d7b
Artifacts:
filename: parliamentdefect.jpg
verbose: true
Tainted: False
Filesize: 988KB
Number pixels: 3.457M
Pixels per second: 69.14MB
User time: 0.040u
Elapsed time: 0:01.049
Version: ImageMagick 6.9.7-4 Q16 x86_64 20170114 http://www.imagemagick.org
......@@ -60,9 +60,7 @@ func NewHistogram(convert, resize, remap string, colormap map[string]string, col
return h, nil
}
func (h *Histogram) GetName() string {
return "histogram"
}
func (h *Histogram) Exec(file string, args ...interface{}) (interface{}, error) {
colors := make(map[string]int64)
......
......@@ -11,6 +11,9 @@ import (
"io"
"net"
"net/http"
"net/url"
"regexp"
"runtime"
)
type Server struct {
......@@ -20,6 +23,7 @@ type Server struct {
log *logging.Logger
accesslog io.Writer
services map[string]Service
wsl bool
}
func NewServer(
......@@ -27,6 +31,7 @@ func NewServer(
log *logging.Logger,
accesslog io.Writer,
srvs map[string]Service,
wsl bool,
) (*Server, error) {
host, port, err := net.SplitHostPort(addr)
if err != nil {
......@@ -41,6 +46,7 @@ func NewServer(
log: log,
accesslog: accesslog,
services: srvs,
wsl: wsl,
}
return srv, nil
}
......@@ -48,24 +54,58 @@ func NewServer(
func (s *Server) ListenAndServe(cert, key string) error {
router := mux.NewRouter()
for prefix, serv := range s.services {
router.HandleFunc(fmt.Sprintf("/%s/[path]", prefix), func(writer http.ResponseWriter, request *http.Request) {
vars := mux.Vars(request)
path, ok := vars["path"]
if !ok {
s.DoPanicf(writer, http.StatusNotFound, "invalid url %s. no path given", request.URL.String())
return
}
result, err := serv.Exec(path)
if err != nil {
s.DoPanicf(writer, http.StatusInternalServerError, "cannot create histogram of %s", path)
return
srvRegexp := regexp.MustCompile(`^/([^/]+)/(.+)$`)
router.MatcherFunc(func(r *http.Request, rm *mux.RouteMatch) bool {
matches := srvRegexp.FindSubmatch([]byte(r.URL.Path))
if len(matches) == 0 {
return false
}
rm.Vars = map[string]string{}
prefix := string(matches[1])
rm.Vars["prefix"] = prefix
rm.Vars["path"] = string(matches[2])
// check whether prefix is known
for key, _ := range s.services {
if key == prefix {
return true
}
jw := json.NewEncoder(writer)
jw.Encode(result)
}).Methods("GET")
}
}
return false
}).HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
writer.Header().Set("Content-Type", "application/json")
vars := mux.Vars(request)
prefix, ok := vars["prefix"]
if !ok {
s.DoPanicf(writer, http.StatusNotFound, "invalid url %s. no path given", request.URL.String())
return
}
serv, ok := s.services[prefix]
if !ok {
s.DoPanicf(writer, http.StatusNotFound, "invalid url %s. wrong prefix %s", request.URL.String(), prefix)
return
}
path, ok := vars["path"]
if !ok {
s.DoPanicf(writer, http.StatusNotFound, "invalid url %s. no path given", request.URL.String())
return
}
if runtime.GOOS != "windows" || (runtime.GOOS == "windows" && s.wsl) {
path = `/` + path
}
p, err := url.QueryUnescape(path)
if err != nil {
s.DoPanicf(writer, http.StatusInternalServerError, "cannot unescape path %s: %v", path, err)
return
}
result, err := serv.Exec(p)
if err != nil {
s.DoPanicf(writer, http.StatusInternalServerError, "cannot create %s of %s: %v", prefix, path, err)
return
}
jw := json.NewEncoder(writer)
jw.Encode(result)
}).Methods("GET")
loggedRouter := handlers.LoggingHandler(s.accesslog, router)
addr := net.JoinHostPort(s.host, s.port)
......
......@@ -2,5 +2,4 @@ package service
type Service interface {
Exec(filename string, args... interface{}) (interface{}, error);
GetName() string;
}
package validate
import (
"bytes"
"context"
"github.com/goph/emperror"
"github.com/op/go-logging"
"os/exec"
"strings"
"time"
)
type ValidateAV struct {
log *logging.Logger
ffmpeg string
timeout time.Duration
wsl bool
}
func NewValidateAV(ffmpeg string, timeout time.Duration, wsl bool, log *logging.Logger) (*ValidateAV, error) {
h := &ValidateAV{
ffmpeg: ffmpeg,
timeout: timeout,
wsl: wsl,
log: log,
}
return h, nil
}
func (h *ValidateAV) Exec(file string, args ...interface{}) (interface{}, error) {
colors := make(map[string]int64)
cmdparam := []string{"-v", "warning", "-i", file, "-f", "null", "-"}
cmdfile := h.ffmpeg
if h.wsl {
cmdparam = append([]string{cmdfile}, cmdparam...)
cmdfile = "wsl"
}
var out bytes.Buffer
out.Grow(1024 * 1024) // 1MB size
var errbuf bytes.Buffer
errbuf.Grow(1024 * 1024) // 1MB size
ctx, cancel := context.WithTimeout(context.Background(), h.timeout)
defer cancel()
cmd := exec.CommandContext(ctx, cmdfile, cmdparam...)
// cmd.Stdin = dataOut
cmd.Stdout = &out
cmd.Stderr = &errbuf
h.log.Infof("executing %v %v", cmdfile, cmdparam)
isError := false
if err := cmd.Run(); err != nil {
exiterr, ok := err.(*exec.ExitError)
if ok && exiterr.ExitCode() == 1 {
isError = true
} else {
outStr := out.String()
errstr := strings.TrimSpace(errbuf.String())
return colors, emperror.Wrapf(err, "error executing (%s %s): %v - %v", cmdfile, cmdparam, outStr, errstr)
}
}
// data := out.String()