server functionality added

README.md

@@ -5,7 +5,7 @@ basierend auf einer Colormap, welche im Konfigurationsfile definiert wird.
 ## Installation
 
     go get gitlab.switch.ch/memoriav/memobase-2020/services/histogram
-    go build gitlab.switch.ch/memoriav/memobase-2020/services/histogram
+    go build gitlab.switch.ch/memoriav/memobase-2020/services/histogram/cmd/histogram
 
 ## Start:
 

config.go → cmd/histogram/config.go

@@ -27,16 +27,17 @@ type ConfigImageMagick struct {
 }
 
 type Config struct {
-	Logfile     string
-	Loglevel    string
-	AccessLog   string
-	CertPEM     string
-	KeyPEM      string
-	Addr        string
-	JwtKey      string
-	JwtAlg      []string
-	ImageMagick ConfigImageMagick
-	Colormap    map[string]string
+	Logfile         string
+	Loglevel        string
+	AccessLog       string
+	CertPEM         string
+	KeyPEM          string
+	Addr            string
+	HistogramPrefix string
+	JwtKey          string
+	JwtAlg          []string
+	ImageMagick     ConfigImageMagick
+	Colormap        map[string]string
 }
 
 func LoadConfig(filepath string) Config {

main.go → cmd/histogram/main.go

 package main
 
 import (
-	"bufio"
-	"bytes"
-	"context"
 	"encoding/json"
 	"flag"
 	"fmt"
-	"github.com/goph/emperror"
-	"image"
-	"image/png"
+	"gitlab.switch.ch/memoriav/memobase-2020/services/histogram/pkg/histogram"
+	"gitlab.switch.ch/memoriav/memobase-2020/services/histogram/pkg/service"
 	"io"
 	"log"
 	"os"
-	"os/exec"
 	"path/filepath"
-	"regexp"
-	"strconv"
-	"strings"
-	"time"
 )
 
 func main() {
@@ -28,13 +19,13 @@ func main() {
 
 	var exPath = ""
 	// if configfile not found try path of executable as prefix
-	if !FileExists(*configFile) {
+	if !service.FileExists(*configFile) {
 		ex, err := os.Executable()
 		if err != nil {
 			panic(err)
 		}
 		exPath = filepath.Dir(ex)
-		if FileExists(filepath.Join(exPath, *configFile)) {
+		if service.FileExists(filepath.Join(exPath, *configFile)) {
 			*configFile = filepath.Join(exPath, *configFile)
 		} else {
 			log.Fatalf("cannot find configuration file: %v", *configFile)
@@ -46,7 +37,7 @@ func main() {
 	config = LoadConfig(*configFile)
 
 	// create logger instance
-	log, lf := CreateLogger("indexer", config.Logfile, config.Loglevel)
+	log, lf := service.CreateLogger("indexer", config.Logfile, config.Loglevel)
 	defer lf.Close()
 
 	var accesslog io.Writer
@@ -63,108 +54,26 @@ func main() {
 	}
 	fmt.Sprintf("%v", accesslog)
 
-	// if necessary create a colormap image
-	if !FileExists(config.ImageMagick.Remap) {
-		width := len(config.Colormap)
-		height := 1
-		upLeft := image.Point{0, 0}
-		lowRight := image.Point{width, height}
-		img := image.NewRGBA(image.Rectangle{upLeft, lowRight})
-		x := 0
-		for name, value := range config.Colormap {
-			color, err := ParseHexColor(value)
-			if err != nil {
-				log.Panicf("cannot parse color %s: %s", name, value)
-				return
-			}
-			img.Set(x, 0, color)
-			x++
-		}
-		f, err := os.Create(config.ImageMagick.Remap)
-		if err != nil {
-			log.Panicf("cannot create file %s: %v", config.ImageMagick.Remap, err)
-			return
-		}
-		png.Encode(f, img)
-		f.Close()
-	}
-	histogram, err := getHistogram(
+	hist, err := histogram.NewHistogram(
 		config.ImageMagick.Convert,
 		config.ImageMagick.Resize,
 		config.ImageMagick.Remap,
+		config.Colormap,
 		config.ImageMagick.Colors,
-		*imgFile,
 		config.ImageMagick.Timeout.Duration,
 		config.ImageMagick.Wsl)
+
+	// if necessary create a colormap image
+	result, err := hist.Exec(*imgFile)
 	if err != nil {
 		log.Panicf("error getting histogram: %v", err)
 		return
 	}
-	result := make(map[string]int64)
-	for col, weight := range histogram {
-		ok := false
-		for name, hex := range config.Colormap {
-			if col == hex {
-				ok = true
-				result[name] = weight
-			}
-		}
-		if !ok {
-			log.Errorf("color %s not in colormap", col)
-		}
-	}
 
-	json, err := json.MarshalIndent(result, "", " ")
+	json, err := json.Marshal(result)
 	if err != nil {
 		log.Errorf("cannot marshal result: %v", err)
 	}
 	fmt.Println(string(json))
 }
 
-func getHistogram(convert, resize, remap string, colors int, file string, timeout time.Duration, wsl bool) (map[string]int64, error) {
-	result := make(map[string]int64)
-
-	cmdparam := []string{file, "-resize", resize, "-dither", "Riemersma", "-colors", fmt.Sprintf("%d", colors), "+dither", "-remap", remap, "-format", `%c`, "histogram:info:"}
-	cmdfile := convert
-	if wsl {
-		cmdparam = append([]string{cmdfile}, cmdparam...)
-		cmdfile = "wsl"
-	}
-
-	var out bytes.Buffer
-	out.Grow(1024 * 1024) // 1MB size
-	ctx, cancel := context.WithTimeout(context.Background(), timeout)
-	defer cancel()
-	cmd := exec.CommandContext(ctx, cmdfile, cmdparam...)
-	//	cmd.Stdin = dataOut
-	cmd.Stdout = &out
-
-	if err := cmd.Run(); err != nil {
-		return result, emperror.Wrapf(err, "error executing (%s %s): %v - %v", cmdfile, cmdparam, out.String(), err)
-	}
-
-	data := out.String()
-	//      44391: (  0,  0,  0,  0) #00000000 none
-	//       182: (  0,  0,  0,  2) #00000002 srgba(0,0,0,0.00784314)
-	scanner := bufio.NewScanner(strings.NewReader(data))
-	r := regexp.MustCompile(` ([0-9]+):.+(#[0-9A-F]{6})[0-9A-F]{2} `)
-	for scanner.Scan() {
-		line := scanner.Text()
-		matches := r.FindStringSubmatch(line)
-		if matches == nil {
-			continue
-		}
-		col := matches[2]
-		countstr := matches[1]
-		if _, ok := result[col]; !ok {
-			result[col] = 0
-		}
-		count, err := strconv.ParseInt(countstr, 10, 64)
-		if err != nil {
-			return result, emperror.Wrapf(err, "cannot parse number %s in line %s", countstr, line)
-		}
-		result[col] += count
-	}
-
-	return result, nil
-}

histogram.toml → configs/histogram.toml

 logfile = "" # log file location
 loglevel = "DEBUG" # CRITICAL|ERROR|WARNING|NOTICE|INFO|DEBUG
 accesslog = "" # http access log file
-#addr = "localhost:81"
+addr = "localhost:83"
 #certpem = "" # tls client certificate file in PEM format
 #keypem = "" # tls client key file in PEM format
-#jwtkey = "swordfish"
-#jwtalg = ["HS256", "HS384", "HS512"] # "hs256" "hs384" "hs512" "es256" "es384" "es512" "ps256" "ps384" "ps512"
+histogramprefix = "histogram"
 
 [ImageMagick]
     #identify = "/usr/bin/identify"

pkg/histogram/service.go

+package histogram
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"fmt"
+	"github.com/goph/emperror"
+	"gitlab.switch.ch/memoriav/memobase-2020/services/histogram/pkg/service"
+	"image"
+	"image/png"
+	"os"
+	"os/exec"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type Histogram struct {
+	convert, resize, remap string
+	colors                 int
+	colormap               map[string]string
+	timeout                time.Duration
+	wsl                    bool
+}
+
+func NewHistogram(convert, resize, remap string, colormap map[string]string, colors int, timeout time.Duration, wsl bool) (*Histogram, error) {
+	h := &Histogram{
+		convert:  convert,
+		resize:   resize,
+		remap:    remap,
+		colors:   colors,
+		colormap: colormap,
+		timeout:  timeout,
+		wsl:      wsl,
+	}
+	if !service.FileExists(remap) {
+		width := len(colormap)
+		height := 1
+		upLeft := image.Point{0, 0}
+		lowRight := image.Point{width, height}
+		img := image.NewRGBA(image.Rectangle{upLeft, lowRight})
+		x := 0
+		for name, value := range colormap {
+			color, err := service.ParseHexColor(value)
+			if err != nil {
+				return nil, emperror.Wrapf(err, "cannot parse color %s: %s", name, value)
+			}
+			img.Set(x, 0, color)
+			x++
+		}
+		f, err := os.Create(remap)
+		if err != nil {
+			return nil, emperror.Wrapf(err, "cannot create file %s", remap)
+		}
+		png.Encode(f, img)
+		f.Close()
+	}
+
+	return h, nil
+}
+func (h *Histogram) GetName() string {
+	return "histogram"
+}
+func (h *Histogram) Exec(file string, args ...interface{}) (interface{}, error) {
+	colors := make(map[string]int64)
+
+	cmdparam := []string{file, "-resize", h.resize, "-dither", "Riemersma", "-colors", fmt.Sprintf("%d", h.colors), "+dither", "-remap", h.remap, "-format", `%c`, "histogram:info:"}
+	cmdfile := h.convert
+	if h.wsl {
+		cmdparam = append([]string{cmdfile}, cmdparam...)
+		cmdfile = "wsl"
+	}
+
+	var out bytes.Buffer
+	out.Grow(1024 * 1024) // 1MB size
+	ctx, cancel := context.WithTimeout(context.Background(), h.timeout)
+	defer cancel()
+	cmd := exec.CommandContext(ctx, cmdfile, cmdparam...)
+	//	cmd.Stdin = dataOut
+	cmd.Stdout = &out
+
+	if err := cmd.Run(); err != nil {
+		outStr := out.String()
+		return colors, emperror.Wrapf(err, "error executing (%s %s): %v", cmdfile, cmdparam, outStr)
+	}
+
+	data := out.String()
+	//      44391: (  0,  0,  0,  0) #00000000 none
+	//       182: (  0,  0,  0,  2) #00000002 srgba(0,0,0,0.00784314)
+	scanner := bufio.NewScanner(strings.NewReader(data))
+	r := regexp.MustCompile(` ([0-9]+):.+(#[0-9A-F]{6})[0-9A-F]{0,2} `)
+	for scanner.Scan() {
+		line := scanner.Text()
+		matches := r.FindStringSubmatch(line)
+		if matches == nil {
+			continue
+		}
+		col := matches[2]
+		countstr := matches[1]
+		if _, ok := colors[col]; !ok {
+			colors[col] = 0
+		}
+		count, err := strconv.ParseInt(countstr, 10, 64)
+		if err != nil {
+			return colors, emperror.Wrapf(err, "cannot parse number %s in line %s", countstr, line)
+		}
+		colors[col] += count
+	}
+
+	result := make(map[string]int64)
+	for col, weight := range colors {
+		ok := false
+		for name, hex := range h.colormap {
+			if col == hex {
+				ok = true
+				result[name] = weight
+			}
+		}
+		if !ok {
+			return nil, fmt.Errorf("color %s not in colormap", col)
+		}
+	}
+
+	return result, nil
+}

helper.go → pkg/service/helper.go

-package main
+package service
 
 import (
 	"fmt"

pkg/service/server.go

+package service
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"github.com/goph/emperror"
+	"github.com/gorilla/handlers"
+	"github.com/gorilla/mux"
+	"github.com/op/go-logging"
+	"io"
+	"net"
+	"net/http"
+)
+
+type Server struct {
+	srv       *http.Server
+	host      string
+	port      string
+	log       *logging.Logger
+	accesslog io.Writer
+	services  map[string]Service
+}
+
+func NewServer(
+	addr string,
+	log *logging.Logger,
+	accesslog io.Writer,
+	srvs map[string]Service,
+) (*Server, error) {
+	host, port, err := net.SplitHostPort(addr)
+	if err != nil {
+		//log.Panicf("cannot split address %s: %v", addr, err)
+		return nil, emperror.Wrapf(err, "cannot split address %s", addr)
+	}
+
+	srv := &Server{
+		srv:       nil,
+		host:      host,
+		port:      port,
+		log:       log,
+		accesslog: accesslog,
+		services:  srvs,
+	}
+	return srv, nil
+}
+
+func (s *Server) ListenAndServe(cert, key string) error {
+	router := mux.NewRouter()
+
+	for prefix, serv := range s.services {
+
+		router.HandleFunc(fmt.Sprintf("/%s/[path]", prefix), func(writer http.ResponseWriter, request *http.Request) {
+			vars := mux.Vars(request)
+			path, ok := vars["path"]
+			if !ok {
+				s.DoPanicf(writer, http.StatusNotFound, "invalid url %s. no path given", request.URL.String())
+				return
+			}
+			result, err := serv.Exec(path)
+			if err != nil {
+				s.DoPanicf(writer, http.StatusInternalServerError, "cannot create histogram of %s", path)
+				return
+			}
+			jw := json.NewEncoder(writer)
+			jw.Encode(result)
+		}).Methods("GET")
+	}
+
+	loggedRouter := handlers.LoggingHandler(s.accesslog, router)
+	addr := net.JoinHostPort(s.host, s.port)
+	s.srv = &http.Server{
+		Handler: loggedRouter,
+		Addr:    addr,
+	}
+	if cert != "" && key != "" {
+		s.log.Infof("starting HTTPS histogram server at https://%v", addr)
+		return s.srv.ListenAndServeTLS(cert, key)
+	} else {
+		s.log.Infof("starting HTTP histogram server at http://%v", addr)
+		return s.srv.ListenAndServe()
+	}
+}
+
+func (s *Server) DoPanicf(writer http.ResponseWriter, status int, message string, a ...interface{}) (err error) {
+	msg := fmt.Sprintf(message, a...)
+	s.DoPanic(writer, status, msg)
+	return
+}
+
+func (s *Server) DoPanic(writer http.ResponseWriter, status int, message string) (err error) {
+	type errData struct {
+		Status     int
+		StatusText string
+		Message    string
+	}
+	s.log.Error(message)
+	data := errData{
+		Status:     status,
+		StatusText: http.StatusText(status),
+		Message:    message,
+	}
+	writer.WriteHeader(status)
+	// if there'ms no error Template, there's no help...
+	jw := json.NewEncoder(writer)
+	jw.Encode(data)
+	return
+}
+
+func (s *Server) Shutdown(ctx context.Context) error {
+	return s.srv.Shutdown(ctx)
+}

pkg/service/service.go

+package service
+
+type Service interface {
+	Exec(filename string, args... interface{}) (interface{}, error);
+	GetName() string;
+}