webservice active

cmd/histogram/config.go

@@ -19,25 +19,47 @@ func (d *duration) UnmarshalText(text []byte) error {
 type ConfigImageMagick struct {
 	Identify string
 	Convert  string
-	Wsl      bool
+}
+
+type ConfigFFMPEG struct {
+	FFMPEG  string
+	FFPROBE string
+}
+
+type ConfigHistogram struct {
+	Prefix   string
+	Colormap map[string]string
 	Timeout  duration
 	Remap    string
 	Colors   int
 	Resize   string
 }
 
+type ConfigValidateImage struct {
+	Prefix  string
+	Timeout duration
+}
+
+type ConfigValidateAV struct {
+	Prefix  string
+	Timeout duration
+}
+
 type Config struct {
-	Logfile         string
-	Loglevel        string
-	AccessLog       string
-	CertPEM         string
-	KeyPEM          string
-	Addr            string
-	HistogramPrefix string
-	JwtKey          string
-	JwtAlg          []string
-	ImageMagick     ConfigImageMagick
-	Colormap        map[string]string
+	Logfile       string
+	Loglevel      string
+	AccessLog     string
+	CertPEM       string
+	KeyPEM        string
+	Addr          string
+	JwtKey        string
+	JwtAlg        []string
+	ImageMagick   ConfigImageMagick
+	ConfigFFMPEG  ConfigFFMPEG
+	Histogram     ConfigHistogram
+	ValidateImage ConfigValidateImage
+	ValidateAV    ConfigValidateAV
+	Wsl           bool
 }
 
 func LoadConfig(filepath string) Config {

cmd/histogram/main.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"gitlab.switch.ch/memoriav/memobase-2020/services/histogram/pkg/histogram"
 	"gitlab.switch.ch/memoriav/memobase-2020/services/histogram/pkg/service"
-	"io"
 	"log"
 	"os"
 	"path/filepath"
@@ -40,28 +39,15 @@ func main() {
 	log, lf := service.CreateLogger("indexer", config.Logfile, config.Loglevel)
 	defer lf.Close()
 
-	var accesslog io.Writer
-	if config.AccessLog == "" {
-		accesslog = os.Stdout
-	} else {
-		f, err := os.OpenFile(config.AccessLog, os.O_WRONLY|os.O_CREATE, 0755)
-		if err != nil {
-			log.Panicf("cannot open file %s: %v", config.AccessLog, err)
-			return
-		}
-		defer f.Close()
-		accesslog = f
-	}
-	fmt.Sprintf("%v", accesslog)
 
 	hist, err := histogram.NewHistogram(
 		config.ImageMagick.Convert,
-		config.ImageMagick.Resize,
-		config.ImageMagick.Remap,
-		config.Colormap,
-		config.ImageMagick.Colors,
-		config.ImageMagick.Timeout.Duration,
-		config.ImageMagick.Wsl)
+		config.Histogram.Resize,
+		config.Histogram.Remap,
+		config.Histogram.Colormap,
+		config.Histogram.Colors,
+		config.Histogram.Timeout.Duration,
+		config.Wsl)
 
 	// if necessary create a colormap image
 	result, err := hist.Exec(*imgFile)

cmd/webservice/config.go

+package main
+
+import (
+	"github.com/BurntSushi/toml"
+	"log"
+	"time"
+)
+
+type duration struct {
+	Duration time.Duration
+}
+
+func (d *duration) UnmarshalText(text []byte) error {
+	var err error
+	d.Duration, err = time.ParseDuration(string(text))
+	return err
+}
+
+type ConfigImageMagick struct {
+	Identify string
+	Convert  string
+}
+
+type ConfigFFMPEG struct {
+	FFMPEG  string
+	FFPROBE string
+}
+
+type ConfigHistogram struct {
+	Prefix   string
+	Colormap map[string]string
+	Timeout  duration
+	Remap    string
+	Colors   int
+	Resize   string
+}
+
+type ConfigValidateImage struct {
+	Prefix  string
+	Timeout duration
+}
+
+type ConfigValidateAV struct {
+	Prefix  string
+	Timeout duration
+}
+
+type Config struct {
+	Logfile       string
+	Loglevel      string
+	AccessLog     string
+	CertPEM       string
+	KeyPEM        string
+	Addr          string
+	JwtKey        string
+	JwtAlg        []string
+	ImageMagick   ConfigImageMagick
+	FFMPEG        ConfigFFMPEG
+	Histogram     ConfigHistogram
+	ValidateImage ConfigValidateImage
+	ValidateAV    ConfigValidateAV
+	Wsl           bool
+}
+
+func LoadConfig(filepath string) Config {
+	var conf Config
+	_, err := toml.DecodeFile(filepath, &conf)
+	if err != nil {
+		log.Fatalln("Error on loading config: ", err)
+	}
+	return conf
+}

cmd/webservice/main.go

+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"gitlab.switch.ch/memoriav/memobase-2020/services/histogram/pkg/histogram"
+	"gitlab.switch.ch/memoriav/memobase-2020/services/histogram/pkg/service"
+	"gitlab.switch.ch/memoriav/memobase-2020/services/histogram/pkg/validate"
+	"io"
+	"log"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"syscall"
+	"time"
+)
+
+func main() {
+	configFile := flag.String("cfg", "./histogram.toml", "config file location")
+	flag.Parse()
+
+	var exPath = ""
+	// if configfile not found try path of executable as prefix
+	if !service.FileExists(*configFile) {
+		ex, err := os.Executable()
+		if err != nil {
+			panic(err)
+		}
+		exPath = filepath.Dir(ex)
+		if service.FileExists(filepath.Join(exPath, *configFile)) {
+			*configFile = filepath.Join(exPath, *configFile)
+		} else {
+			log.Fatalf("cannot find configuration file: %v", *configFile)
+			return
+		}
+	}
+	// configfile should exists at this place
+	var config Config
+	config = LoadConfig(*configFile)
+
+	// create logger instance
+	log, lf := service.CreateLogger("indexer", config.Logfile, config.Loglevel)
+	defer lf.Close()
+
+	var accesslog io.Writer
+	if config.AccessLog == "" {
+		accesslog = os.Stdout
+	} else {
+		f, err := os.OpenFile(config.AccessLog, os.O_WRONLY|os.O_CREATE, 0755)
+		if err != nil {
+			log.Panicf("cannot open file %s: %v", config.AccessLog, err)
+			return
+		}
+		defer f.Close()
+		accesslog = f
+	}
+	fmt.Sprintf("%v", accesslog)
+
+	imgval, err := validate.NewValidateImage(config.ImageMagick.Identify, config.ValidateImage.Timeout.Duration, config.Wsl, log)
+	if err != nil {
+		log.Fatalf("cannot initialize ValidateImage: %v", err)
+		return
+	}
+
+	avval, err := validate.NewValidateAV(config.FFMPEG.FFMPEG, config.ValidateAV.Timeout.Duration, config.Wsl, log)
+	if err != nil {
+		log.Fatalf("cannot initialize ValidateAV: %v", err)
+		return
+	}
+
+	hist, err := histogram.NewHistogram(
+		config.ImageMagick.Convert,
+		config.Histogram.Resize,
+		config.Histogram.Remap,
+		config.Histogram.Colormap,
+		config.Histogram.Colors,
+		config.Histogram.Timeout.Duration,
+		config.Wsl)
+	if err != nil {
+		log.Fatalf("cannot initialize histogram: %v", err)
+		return
+	}
+
+	srv, err := service.NewServer(
+		config.Addr,
+		log,
+		accesslog,
+		map[string]service.Service{
+			config.Histogram.Prefix:     hist,
+			config.ValidateImage.Prefix: imgval,
+			config.ValidateAV.Prefix:    avval,
+		},
+		config.Wsl,
+	)
+	if err != nil {
+		log.Fatalf("cannot initialize server: %v", err)
+		return
+	}
+	go func() {
+		if err := srv.ListenAndServe(config.CertPEM, config.KeyPEM); err != nil {
+			log.Errorf("server died: %v", err)
+		}
+	}()
+
+	end := make(chan bool, 1)
+
+	// process waiting for interrupt signal (TERM or KILL)
+	go func() {
+		sigint := make(chan os.Signal, 1)
+
+		// interrupt signal sent from terminal
+		signal.Notify(sigint, os.Interrupt)
+
+		signal.Notify(sigint, syscall.SIGTERM)
+		signal.Notify(sigint, syscall.SIGKILL)
+
+		<-sigint
+
+		// We received an interrupt signal, shut down.
+		log.Infof("shutdown requested")
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+
+		srv.Shutdown(ctx)
+
+		end <- true
+	}()
+
+	<-end
+	log.Info("server stopped")
+}

configs/histogram.toml

@@ -4,12 +4,26 @@ accesslog = "" # http access log file
 addr = "localhost:83"
 #certpem = "" # tls client certificate file in PEM format
 #keypem = "" # tls client key file in PEM format
-histogramprefix = "histogram"
+wsl = true  # true, if executable is within linux subsystem on windows
 
 [ImageMagick]
-    #identify = "/usr/bin/identify"
+    identify = "/usr/bin/identify"
     convert = "/usr/bin/convert"
-    wsl = true  # true, if executable is within linux subsystem on windows
+
+[FFMPEG]
+    ffmpeg = "/usr/local/bin/ffmpeg2"
+    ffprobe = "/usr/local/bin/ffprobe2"
+
+[ValidateImage]
+    prefix = "validateimage"
+    timeout = "10s"
+
+[ValidateAV]
+    prefix = "validateav"
+    timeout = "10s"
+
+[Histogram]
+    prefix = "histogram"
     timeout = "10s"
     remap = "color.png" # das Bild mit der Zielfarbpalette. Wird erzeugt, falls nicht vorhanden. (Schreibrechte beachten)
     colors = 10  # Anzahl der Farben im Histogramm

out.dat

+Image: parliamentdefect.jpg
+  Format: JPEG (Joint Photographic Experts Group JFIF format)
+  Mime type: image/jpeg
+  Class: DirectClass
+  Geometry: 2480x1394+0+0
+  Resolution: 150x150
+  Print size: 16.5333x9.29333
+  Units: PixelsPerInch
+  Type: TrueColor
+  Endianess: Undefined
+  Colorspace: sRGB
+  Depth: 8-bit
+  Channel depth:
+    red: 8-bit
+    green: 8-bit
+    blue: 8-bit
+  Channel statistics:
+    Pixels: 3457120
+    Red:
+      min: 0 (0)
+      max: 255 (1)
+      mean: 99.1014 (0.388633)
+      standard deviation: 62.826 (0.246376)
+      kurtosis: -0.177222
+      skewness: 1.00838
+      entropy: 0.922141
+    Green:
+      min: 0 (0)
+      max: 255 (1)
+      mean: 84.6732 (0.332052)
+      standard deviation: 58.3896 (0.228979)
+      kurtosis: 0.773212
+      skewness: 1.30255
+      entropy: 0.893161
+    Blue:
+      min: 0 (0)
+      max: 255 (1)
+      mean: 78.7253 (0.308727)
+      standard deviation: 56.2547 (0.220607)
+      kurtosis: 1.76577
+      skewness: 1.60893
+      entropy: 0.882406
+  Image statistics:
+    Overall:
+      min: 0 (0)
+      max: 255 (1)
+      mean: 87.5 (0.343137)
+      standard deviation: 59.2201 (0.232235)
+      kurtosis: 0.767738
+      skewness: 1.319
+      entropy: 0.899236
+  Rendering intent: Perceptual
+  Gamma: 0.454545
+  Chromaticity:
+    red primary: (0.64,0.33)
+    green primary: (0.3,0.6)
+    blue primary: (0.15,0.06)
+    white point: (0.3127,0.329)
+  Background color: white
+  Border color: srgb(223,223,223)
+  Matte color: grey74
+  Transparent color: black
+  Interlace: None
+  Intensity: Undefined
+  Compose: Over
+  Page geometry: 2480x1394+0+0
+  Dispose: Undefined
+  Iterations: 0
+  Compression: JPEG
+  Quality: 90
+  Orientation: Undefined
+  Properties:
+    date:create: 2020-06-18T12:11:31+02:00
+    date:modify: 2020-06-18T12:11:31+02:00
+    jpeg:colorspace: 2
+    jpeg:sampling-factor: 2x2,1x1,1x1
+    signature: 0c81f2574d32dda8581b104c2f33c751ab2b9e43ad2335dcc195acd95df73d7b
+  Artifacts:
+    filename: parliamentdefect.jpg
+    verbose: true
+  Tainted: False
+  Filesize: 988KB
+  Number pixels: 3.457M
+  Pixels per second: 69.14MB
+  User time: 0.040u
+  Elapsed time: 0:01.049
+  Version: ImageMagick 6.9.7-4 Q16 x86_64 20170114 http://www.imagemagick.org

pkg/histogram/service.go

@@ -60,9 +60,7 @@ func NewHistogram(convert, resize, remap string, colormap map[string]string, col
 
 	return h, nil
 }
-func (h *Histogram) GetName() string {
-	return "histogram"
-}
+
 func (h *Histogram) Exec(file string, args ...interface{}) (interface{}, error) {
 	colors := make(map[string]int64)
 

pkg/service/server.go

@@ -11,6 +11,9 @@ import (
 	"io"
 	"net"
 	"net/http"
+	"net/url"
+	"regexp"
+	"runtime"
 )
 
 type Server struct {
@@ -20,6 +23,7 @@ type Server struct {
 	log       *logging.Logger
 	accesslog io.Writer
 	services  map[string]Service
+	wsl       bool
 }
 
 func NewServer(
@@ -27,6 +31,7 @@ func NewServer(
 	log *logging.Logger,
 	accesslog io.Writer,
 	srvs map[string]Service,
+	wsl bool,
 ) (*Server, error) {
 	host, port, err := net.SplitHostPort(addr)
 	if err != nil {
@@ -41,6 +46,7 @@ func NewServer(
 		log:       log,
 		accesslog: accesslog,
 		services:  srvs,
+		wsl:       wsl,
 	}
 	return srv, nil
 }
@@ -48,24 +54,58 @@ func NewServer(
 func (s *Server) ListenAndServe(cert, key string) error {
 	router := mux.NewRouter()
 
-	for prefix, serv := range s.services {
-
-		router.HandleFunc(fmt.Sprintf("/%s/[path]", prefix), func(writer http.ResponseWriter, request *http.Request) {
-			vars := mux.Vars(request)
-			path, ok := vars["path"]
-			if !ok {
-				s.DoPanicf(writer, http.StatusNotFound, "invalid url %s. no path given", request.URL.String())
-				return
-			}
-			result, err := serv.Exec(path)
-			if err != nil {
-				s.DoPanicf(writer, http.StatusInternalServerError, "cannot create histogram of %s", path)
-				return
+	srvRegexp := regexp.MustCompile(`^/([^/]+)/(.+)$`)
+	router.MatcherFunc(func(r *http.Request, rm *mux.RouteMatch) bool {
+		matches := srvRegexp.FindSubmatch([]byte(r.URL.Path))
+		if len(matches) == 0 {
+			return false
+		}
+		rm.Vars = map[string]string{}
+		prefix := string(matches[1])
+		rm.Vars["prefix"] = prefix
+		rm.Vars["path"] = string(matches[2])
+		// check whether prefix is known
+		for key, _ := range s.services {
+			if key == prefix {
+				return true
 			}
-			jw := json.NewEncoder(writer)
-			jw.Encode(result)
-		}).Methods("GET")
-	}
+		}
+		return false
+	}).HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
+		writer.Header().Set("Content-Type", "application/json")
+
+		vars := mux.Vars(request)
+		prefix, ok := vars["prefix"]
+		if !ok {
+			s.DoPanicf(writer, http.StatusNotFound, "invalid url %s. no path given", request.URL.String())
+			return
+		}
+		serv, ok := s.services[prefix]
+		if !ok {
+			s.DoPanicf(writer, http.StatusNotFound, "invalid url %s. wrong prefix %s", request.URL.String(), prefix)
+			return
+		}
+		path, ok := vars["path"]
+		if !ok {
+			s.DoPanicf(writer, http.StatusNotFound, "invalid url %s. no path given", request.URL.String())
+			return
+		}
+		if runtime.GOOS != "windows" || (runtime.GOOS == "windows" && s.wsl) {
+			path = `/` + path
+		}
+		p, err := url.QueryUnescape(path)
+		if err != nil {
+			s.DoPanicf(writer, http.StatusInternalServerError, "cannot unescape path %s: %v", path, err)
+			return
+		}
+		result, err := serv.Exec(p)
+		if err != nil {
+			s.DoPanicf(writer, http.StatusInternalServerError, "cannot create %s of %s: %v", prefix, path, err)
+			return
+		}
+		jw := json.NewEncoder(writer)
+		jw.Encode(result)
+	}).Methods("GET")
 
 	loggedRouter := handlers.LoggingHandler(s.accesslog, router)
 	addr := net.JoinHostPort(s.host, s.port)

pkg/service/service.go

@@ -2,5 +2,4 @@ package service
 
 type Service interface {
 	Exec(filename string, args... interface{}) (interface{}, error);
-	GetName() string;
 }

pkg/validate/av.go

+package validate
+
+import (
+	"bytes"
+	"context"
+	"github.com/goph/emperror"
+	"github.com/op/go-logging"
+	"os/exec"
+	"strings"
+	"time"
+)
+
+type ValidateAV struct {
+	log *logging.Logger
+	ffmpeg  string
+	timeout time.Duration
+	wsl     bool
+}
+
+func NewValidateAV(ffmpeg string, timeout time.Duration, wsl bool, log *logging.Logger) (*ValidateAV, error) {
+	h := &ValidateAV{
+		ffmpeg:  ffmpeg,
+		timeout: timeout,
+		wsl:     wsl,
+		log: log,
+	}
+
+	return h, nil
+}
+
+func (h *ValidateAV) Exec(file string, args ...interface{}) (interface{}, error) {
+	colors := make(map[string]int64)
+
+	cmdparam := []string{"-v", "warning", "-i", file, "-f", "null", "-"}
+	cmdfile := h.ffmpeg
+	if h.wsl {
+		cmdparam = append([]string{cmdfile}, cmdparam...)
+		cmdfile = "wsl"
+	}
+
+	var out bytes.Buffer
+	out.Grow(1024 * 1024) // 1MB size
+	var errbuf bytes.Buffer
+	errbuf.Grow(1024 * 1024) // 1MB size
+	ctx, cancel := context.WithTimeout(context.Background(), h.timeout)
+	defer cancel()
+	cmd := exec.CommandContext(ctx, cmdfile, cmdparam...)
+	//	cmd.Stdin = dataOut
+	cmd.Stdout = &out
+	cmd.Stderr = &errbuf
+
+	h.log.Infof("executing %v %v", cmdfile, cmdparam)
+
+	isError := false
+	if err := cmd.Run(); err != nil {
+		exiterr, ok := err.(*exec.ExitError)
+		if ok && exiterr.ExitCode() == 1 {
+			isError = true
+		} else {
+			outStr := out.String()
+			errstr := strings.TrimSpace(errbuf.String())
+			return colors, emperror.Wrapf(err, "error executing (%s %s): %v - %v", cmdfile, cmdparam, outStr, errstr)
+		}
+	}
+
+	//	data := out.String()
+	errstr := strings.TrimSpace(errbuf.String())