Commit 577cab7e authored by Jürgen Enge's avatar Jürgen Enge

database table in config

parent 04eab53e
......@@ -42,6 +42,7 @@ type Crawler struct {
PageSize int
HeaderSize int
Workers int
Table string
}
type Config struct {
......
......@@ -106,6 +106,8 @@ func main() {
db,
config.Crawler.Workers,
config.Crawler.PageSize,
config.DB.Schema,
config.Crawler.Table,
config.TempDir,
config.Siegfried,
config.Crawler.OK.Duration,
......
......@@ -12,13 +12,14 @@ ffprobe = "/usr/local/bin/ffprobe2"
[crawler]
headersize = 5000 # number of bytes which are send to siegfried
pagesize = 500 # number of entries done by one database access
pagesize = 1000 # number of entries done by one database access
ok = "600h" # check files every 600 hours
error = "168h" # if there's an error, check all 168 hours minimum
errornew = "22h" # new errors should be checked the next day
cron = "42 * * * *" # cron format (https://pkg.go.dev/github.com/robfig/cron?tab=doc)
workers = 4
workers = 7
timeout = "5s"
table = "test2"
[banner]
folder = "c:/temp/banner"
......
......@@ -15,6 +15,8 @@ type Crawler struct {
log *logging.Logger
workers int
pageSize int
schema string
table string
tempDir string
siegfried Siegfried
crawlOK time.Duration
......@@ -35,6 +37,8 @@ func NewCrawler(
db *sql.DB,
workers int,
pageSize int,
schema string,
table string,
tempDir, siegfried string,
crawlOK, crawlError, crawlErrorNew time.Duration,
ffmpeg, ffprobe string,
......@@ -44,6 +48,8 @@ func NewCrawler(
db: db,
workers: workers,
pageSize: pageSize,
schema: schema,
table: table,
tempDir: tempDir,
siegfried: Siegfried{surl: siegfried},
crawlOK: crawlOK,
......@@ -177,16 +183,16 @@ func (cr *Crawler) CrawlNew() error {
}
/*
if err := cr.checkList(entries); err != nil {
return emperror.Wrapf(err, "cannot linkCheck result list")
}
*/
if err := cr.checkList(entries); err != nil {
return emperror.Wrapf(err, "cannot linkCheck result list")
}
*/
// wait until last worker ist done
for {
if cr.jobQueue.isIdle() {
break;
break
}
time.Sleep(1*time.Second)
time.Sleep(1 * time.Second)
}
}
return nil
......@@ -222,9 +228,9 @@ func (cr *Crawler) CrawlError() error {
// wait until last worker ist done
for {
if cr.jobQueue.isIdle() {
break;
break
}
time.Sleep(1*time.Second)
time.Sleep(1 * time.Second)
}
}
return nil
......@@ -259,9 +265,9 @@ func (cr *Crawler) CrawlOK() error {
// wait until last worker ist done
for {
if cr.jobQueue.isIdle() {
break;
break
}
time.Sleep(1*time.Second)
time.Sleep(1 * time.Second)
}
}
return nil
......
......@@ -2,6 +2,7 @@ package memocrawler
import (
"encoding/json"
"fmt"
"github.com/goph/emperror"
"gitlab.switch.ch/memoriav/memobase-2020/streaming-server/memostream"
)
......@@ -57,7 +58,7 @@ func (w Worker) linkcheck( entry *memostream.MediaEntry ) error {
if err != nil {
return emperror.Wrapf(err, "error checking entry %s", entry.Signature)
}
sqlstr := "UPDATE test.test2 SET lastcheck=NOW()"
sqlstr := fmt.Sprintf("UPDATE %s.%s SET lastcheck=NOW()", w.cr.schema, w.cr.table)
var statusStr string
var params []interface{}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment