database table in config

main/config.go

@@ -42,6 +42,7 @@ type Crawler struct {
 	PageSize   int
 	HeaderSize int
 	Workers    int
+	Table      string
 }
 
 type Config struct {

main/main.go

@@ -106,6 +106,8 @@ func main() {
 		db,
 		config.Crawler.Workers,
 		config.Crawler.PageSize,
+		config.DB.Schema,
+		config.Crawler.Table,
 		config.TempDir,
 		config.Siegfried,
 		config.Crawler.OK.Duration,

memocrawler.toml

@@ -12,13 +12,14 @@ ffprobe = "/usr/local/bin/ffprobe2"
 
 [crawler]
     headersize = 5000 # number of bytes which are send to siegfried
-    pagesize = 500 # number of entries done by one database access
+    pagesize = 1000 # number of entries done by one database access
     ok = "600h" # check files every 600 hours
     error = "168h" # if there's an error, check all 168 hours minimum
     errornew = "22h" # new errors should be checked the next day
     cron = "42 * * * *" # cron format (https://pkg.go.dev/github.com/robfig/cron?tab=doc)
-    workers = 4
+    workers = 7
     timeout = "5s"
+    table = "test2"
 
 [banner]
     folder = "c:/temp/banner"

memocrawler/crawler.go

@@ -15,6 +15,8 @@ type Crawler struct {
 	log           *logging.Logger
 	workers       int
 	pageSize      int
+	schema        string
+	table         string
 	tempDir       string
 	siegfried     Siegfried
 	crawlOK       time.Duration
@@ -35,6 +37,8 @@ func NewCrawler(
 	db *sql.DB,
 	workers int,
 	pageSize int,
+	schema string,
+	table string,
 	tempDir, siegfried string,
 	crawlOK, crawlError, crawlErrorNew time.Duration,
 	ffmpeg, ffprobe string,
@@ -44,6 +48,8 @@ func NewCrawler(
 		db:            db,
 		workers:       workers,
 		pageSize:      pageSize,
+		schema:        schema,
+		table:         table,
 		tempDir:       tempDir,
 		siegfried:     Siegfried{surl: siegfried},
 		crawlOK:       crawlOK,
@@ -177,16 +183,16 @@ func (cr *Crawler) CrawlNew() error {
 		}
 
 		/*
-		if err := cr.checkList(entries); err != nil {
-			return emperror.Wrapf(err, "cannot linkCheck result list")
-		}
-		 */
+			if err := cr.checkList(entries); err != nil {
+				return emperror.Wrapf(err, "cannot linkCheck result list")
+			}
+		*/
 		// wait until last worker ist done
 		for {
 			if cr.jobQueue.isIdle() {
-				break;
+				break
 			}
-			time.Sleep(1*time.Second)
+			time.Sleep(1 * time.Second)
 		}
 	}
 	return nil
@@ -222,9 +228,9 @@ func (cr *Crawler) CrawlError() error {
 		// wait until last worker ist done
 		for {
 			if cr.jobQueue.isIdle() {
-				break;
+				break
 			}
-			time.Sleep(1*time.Second)
+			time.Sleep(1 * time.Second)
 		}
 	}
 	return nil
@@ -259,9 +265,9 @@ func (cr *Crawler) CrawlOK() error {
 		// wait until last worker ist done
 		for {
 			if cr.jobQueue.isIdle() {
-				break;
+				break
 			}
-			time.Sleep(1*time.Second)
+			time.Sleep(1 * time.Second)
 		}
 	}
 	return nil

memocrawler/worker.go

@@ -2,6 +2,7 @@ package memocrawler
 
 import (
 	"encoding/json"
+	"fmt"
 	"github.com/goph/emperror"
 	"gitlab.switch.ch/memoriav/memobase-2020/streaming-server/memostream"
 )
@@ -57,7 +58,7 @@ func (w Worker) linkcheck( entry *memostream.MediaEntry ) error {
 	if err != nil {
 		return emperror.Wrapf(err, "error checking entry %s", entry.Signature)
 	}
-	sqlstr := "UPDATE test.test2 SET lastcheck=NOW()"
+	sqlstr := fmt.Sprintf("UPDATE %s.%s SET lastcheck=NOW()", w.cr.schema, w.cr.table)
 
 	var statusStr string
 	var params []interface{}