Commit c0ccddcf authored by Jürgen Enge's avatar Jürgen Enge

error management optimized

parent e304797d
......@@ -8,7 +8,7 @@ import (
_ "github.com/go-sql-driver/mysql"
"github.com/op/go-logging"
"github.com/robfig/cron"
"gitlab.switch.ch/memoriav/memobase-2020/services/streaming-server/memostream"
"gitlab.switch.ch/memoriav/memobase-2020/services/streaming-server/pkg/memostream"
"gitlab.switch.ch/memoriav/memobase-2020/services/url-checker/pkg/memocrawler"
"log"
"os"
......@@ -129,21 +129,6 @@ func main() {
cl := memocrawler.NewCrawlerLinkcheck(cr, config.Crawler.Timeout.Duration, config.Crawler.HeaderSize)
cr.SetCrawlerLinkcheck(cl)
/*
sqlstr := fmt.Sprintf("INSERT INTO %s.test2 (sig, uri, access, proto, status) VALUES (?, ?, ?, ?, ?)", config.DB.Schema)
for i := 1000; i < 5000; i++ {
var params []interface{}
params = append(params, fmt.Sprintf("sig-%v", i),
"https://ba14ns21403.fhnw.ch/video/open/performance/2002_B_B_Yours_Sincerly.mov.mp4",
"public",
"redirect",
"new")
_, err := db.Exec(sqlstr, params...)
if err != nil {
log.Panicf("%s, %v: %v", sqlstr, params, err)
}
}
*/
if config.Crawler.Cron == "" {
cr.Start()
return
......
......@@ -17,9 +17,9 @@ ffprobe = "/usr/local/bin/ffprobe2"
pagesize = 1000 # number of entries done by one database access
ok = "600h" # check files every 600 hours
error = "168h" # if there's an error, check all 168 hours minimum
errornew = "22h" # new errors should be checked the next day
errornew = "2h" # new errors should be checked the next day
cron = "" # "42 * * * *" # cron format (https://pkg.go.dev/github.com/robfig/cron?tab=doc) starts directly one crawl if empty
workers = 7
workers = 1
timeout = "5s"
[metadata]
......@@ -47,9 +47,11 @@ folder = "c:/temp"
[DB]
#if dsn is empty, the static resolver will be used
#[username[:password]@][protocol[(address)]]/dbname[?param1=value1&...&paramN=valueN]
dsn = "test:HDNQiaqNqu4IjmUPATJr@tcp(localhost:3306)/test"
#dsn = "test:HDNQiaqNqu4IjmUPATJr@tcp(localhost:3306)/test"
dsn = "urlchecker:siex5ieNguuQuei@tcp(localhost:4306)/medienserver"
#dsn = ""
# should be smaller than server connection timeout to allow controlled reconnect
connMaxTimeout = "4h"
schema = "test"
#schema = "test"
schema = "medienserver"
......@@ -12,5 +12,5 @@ require (
github.com/gorilla/mux v1.7.4 // indirect
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
github.com/robfig/cron v1.2.0
gitlab.switch.ch/memoriav/memobase-2020/services/streaming-server v0.0.0-20200428134555-1b2f17c5d83b
gitlab.switch.ch/memoriav/memobase-2020/services/streaming-server v0.0.0-20200429072912-1967cd685b73
)
......@@ -9,7 +9,6 @@ github.com/bugsnag/bugsnag-go v1.4.0/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqR
github.com/bugsnag/panicwrap v1.2.0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE=
github.com/certifi/gocertifi v0.0.0-20190105021004-abcd57078448/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgrijalva/jwt-go v1.0.2 h1:KPldsxuKGsS2FPWsNeg9ZO18aCrGKujPoWXn2yo+KQM=
github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
......@@ -46,6 +45,8 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
gitlab.switch.ch/memoriav/memobase-2020/services/streaming-server v0.0.0-20200428134555-1b2f17c5d83b h1:unWTj5yUDM1n7h7ri/O1p3EM//O1lFtbE3rHzBxGbMI=
gitlab.switch.ch/memoriav/memobase-2020/services/streaming-server v0.0.0-20200428134555-1b2f17c5d83b/go.mod h1:U/TH5AF3w1u1RhRUMym/wzuILayfStGsqPunzCpv7mw=
gitlab.switch.ch/memoriav/memobase-2020/services/streaming-server v0.0.0-20200429072912-1967cd685b73 h1:gRZDEXWyeOEM/J+gyjtxqWGkDBg5Ort7ml91wTC1eO0=
gitlab.switch.ch/memoriav/memobase-2020/services/streaming-server v0.0.0-20200429072912-1967cd685b73/go.mod h1:LCD6NUeRRlC1M0oOvmrQ6FeDQrrHbJY2tzxEsS0VjWg=
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
......
......@@ -16,7 +16,7 @@ import (
"fmt"
"github.com/goph/emperror"
"github.com/op/go-logging"
"gitlab.switch.ch/memoriav/memobase-2020/services/streaming-server/memostream"
"gitlab.switch.ch/memoriav/memobase-2020/services/streaming-server/pkg/memostream"
"time"
)
......@@ -220,11 +220,12 @@ func (cr *Crawler) CrawlError() error {
sqlstr := fmt.Sprintf("SELECT sig AS signature, uri, access, proto AS protocol, `status` "+
"FROM %s.entities "+
"WHERE status='error' "+
"AND lastcheck < ? "+
"AND (lastcheck < ? OR lastchange > ?)"+
"AND lastcheck < TIMESTAMPADD(SECOND, ?, NOW()) "+
"AND (lastcheck < TIMESTAMPADD(SECOND, ?, NOW()) OR lastchange < TIMESTAMPADD(SECOND, ?, NOW())) "+
"AND 1=0 "+
"ORDER BY lastchange ASC", cr.schema)
for {
entries, err := cr.getEntries(sqlstr, cr.crawlErrorNew, cr.crawlError, cr.crawlErrorNew)
entries, err := cr.getEntries(sqlstr, int64(cr.crawlErrorNew/time.Second), int64(cr.crawlError/time.Second), int64(cr.crawlErrorNew/time.Second))
if err != nil {
return emperror.Wrapf(err, "cannot get new entries")
}
......
......@@ -17,7 +17,7 @@ import (
"errors"
"fmt"
"github.com/goph/emperror"
"gitlab.switch.ch/memoriav/memobase-2020/services/streaming-server/memostream"
"gitlab.switch.ch/memoriav/memobase-2020/services/streaming-server/pkg/memostream"
"io"
"io/ioutil"
"mime"
......
......@@ -13,7 +13,7 @@ package memocrawler
import (
"container/list"
"github.com/op/go-logging"
"gitlab.switch.ch/memoriav/memobase-2020/services/streaming-server/memostream"
"gitlab.switch.ch/memoriav/memobase-2020/services/streaming-server/pkg/memostream"
"sync"
"time"
)
......
......@@ -14,7 +14,7 @@ import (
"encoding/json"
"fmt"
"github.com/goph/emperror"
"gitlab.switch.ch/memoriav/memobase-2020/services/streaming-server/memostream"
"gitlab.switch.ch/memoriav/memobase-2020/services/streaming-server/pkg/memostream"
)
type Worker struct {
......@@ -67,8 +67,17 @@ func (w Worker) linkcheck( entry *memostream.MediaEntry ) error {
metadata, mimetype, width, height, duration, err := w.cr.cl.linkCheck(entry, true)
if err != nil {
err := emperror.Wrapf(err, "error checking entry %s", entry.Signature)
sqlstr := fmt.Sprintf("UPDATE %s.entities SET lastcheck=NOW(), status=?, errormessage=?", w.cr.schema)
params := []interface{}{"error", err.Error()}
sqlstr := fmt.Sprintf("UPDATE %s.entities SET lastcheck=NOW(), status=?, errormessage=? WHERE sig=?", w.cr.schema)
params := []interface{}{"error", err.Error(), entry.Signature}
if _, err := w.cr.db.Exec(sqlstr, params...); err != nil {
return emperror.Wrapf(err, "error executing sql %s [%v]", sqlstr, params)
}
return err
}
if mimetype == "" {
err := fmt.Errorf("media not accessible: %v", metadata["errors"])
sqlstr := fmt.Sprintf("UPDATE %s.entities SET lastcheck=NOW(), status=?, errormessage=? WHERE sig=?", w.cr.schema)
params := []interface{}{"error", err.Error(), entry.Signature}
if _, err := w.cr.db.Exec(sqlstr, params...); err != nil {
return emperror.Wrapf(err, "error executing sql %s [%v]", sqlstr, params)
}
......@@ -78,8 +87,8 @@ func (w Worker) linkcheck( entry *memostream.MediaEntry ) error {
metajson, err := json.Marshal(metadata)
if err != nil {
err := emperror.Wrapf(err, "cannot marshal metadata %v", metadata)
sqlstr := fmt.Sprintf("UPDATE %s.entities SET lastcheck=NOW(), status=?, errormessage=?", w.cr.schema)
params := []interface{}{"error", err.Error()}
sqlstr := fmt.Sprintf("UPDATE %s.entities SET lastcheck=NOW(), status=?, errormessage=? WHERE sig=?", w.cr.schema)
params := []interface{}{"error", err.Error(), entry.Signature}
if _, err := w.cr.db.Exec(sqlstr, params...); err != nil {
return emperror.Wrapf(err, "error executing sql %s [%v]", sqlstr, params)
}
......@@ -93,15 +102,15 @@ func (w Worker) linkcheck( entry *memostream.MediaEntry ) error {
if _, err := w.cr.db.Exec(sqlstr, params...); err != nil {
err := emperror.Wrapf(err, "error executing sql %s [%v]", sqlstr, params)
sqlstr := fmt.Sprintf("UPDATE %s.entities SET lastcheck=NOW(), status=?, errormessage=?", w.cr.schema)
params := []interface{}{"error", err.Error()}
sqlstr := fmt.Sprintf("UPDATE %s.entities SET lastcheck=NOW(), status=?, errormessage=? WHERE sig=?", w.cr.schema)
params := []interface{}{"error", err.Error(), entry.Signature}
if _, err := w.cr.db.Exec(sqlstr, params...); err != nil {
return emperror.Wrapf(err, "error executing sql %s [%v]", sqlstr, params)
}
return err
}
sqlstr = fmt.Sprintf("UPDATE %s.entities SET lastcheck=NOW(), status=?, errormessage=?", w.cr.schema)
params = []interface{}{"ok", ""}
sqlstr = fmt.Sprintf("UPDATE %s.entities SET lastcheck=NOW(), status=?, errormessage=? WHERE sig=?", w.cr.schema)
params = []interface{}{"ok", "", entry.Signature}
if _, err := w.cr.db.Exec(sqlstr, params...); err != nil {
return emperror.Wrapf(err, "error executing sql %s [%v]", sqlstr, params)
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment