Commit 9c3bb090 authored by Jürgen Enge's avatar Jürgen Enge

webtimeout added to config

parent 919d48cf
Pipeline #18660 passed with stages
in 1 minute and 49 seconds
......@@ -3,7 +3,7 @@ package main
import (
"github.com/BurntSushi/toml"
"log"
"os"
"os"
"time"
)
......@@ -47,19 +47,20 @@ type Crawler struct {
}
type Config struct {
Logfile string
Loglevel string
CertPEM string
KeyPEM string
Addr string
JwtKey string
JwtAlg []string
DB CfgDBMySQL
TempDir string
Indexer string
Crawler Crawler
Metadata Meta
FileMap []FileMap
Logfile string
Loglevel string
CertPEM string
KeyPEM string
Addr string
JwtKey string
JwtAlg []string
DB CfgDBMySQL
TempDir string
Indexer string
Crawler Crawler
Metadata Meta
FileMap []FileMap
WebTimeout duration
}
func LoadConfig(filepath string) Config {
......@@ -68,9 +69,12 @@ func LoadConfig(filepath string) Config {
if err != nil {
log.Fatalln("Error on loading config: ", err)
}
dsn := os.Getenv("DSN")
if dsn != "" {
conf.DB.Dsn = dsn
}
dsn := os.Getenv("DSN")
if dsn != "" {
conf.DB.Dsn = dsn
}
if conf.WebTimeout.Duration == 0 {
conf.WebTimeout.Duration = 5 * time.Second
}
return conf
}
......@@ -83,6 +83,7 @@ func main() {
config.Metadata.Workers,
config.Metadata.PageSize,
config.Crawler.HeaderSize,
config.WebTimeout.Duration,
config.Indexer,
fm,
log)
......
......@@ -8,6 +8,7 @@ jwtkey = "swordfish"
jwtalg = ["HS256", "HS384", "HS512"] # "hs256" "hs384" "hs512" "es256" "es384" "es512" "ps256" "ps384" "ps512"
tempdir = "C:/temp/"
indexer = "http://localhost:81"
webtimeout = "12s"
[crawler]
headersize = 512 # number of bytes which are read from file
......
......@@ -43,6 +43,7 @@ type Crawler struct {
bannerJobQueue *JobQueue
bannerJobChannel chan *Job
headerSize int
webTimeout time.Duration
}
func NewCrawler(
......@@ -56,9 +57,11 @@ func NewCrawler(
metaWorkers int,
metaPageSize int,
headerSize int,
webTimeout time.Duration,
indexer string,
mapping memostream.Filesystem,
log *logging.Logger) *Crawler {
log *logging.Logger,
) *Crawler {
cr := &Crawler{
db: db,
workers: workers,
......@@ -73,6 +76,7 @@ func NewCrawler(
metaWorkers: metaWorkers,
metaPageSize: metaPageSize,
headerSize: headerSize,
webTimeout: webTimeout,
mapping: mapping,
log: log,
}
......
......@@ -19,7 +19,6 @@ import (
"io"
"net/http"
"net/url"
"time"
)
type Worker struct {
......@@ -29,7 +28,7 @@ type Worker struct {
}
//var w.cr.headerSize int64 = 512
var timeout time.Duration = 5 * time.Second
//var timeout time.Duration = 5 * time.Second
// NewWorker creates takes a numeric id and a channel w/ worker pool.
func NewWorker(id int, cr *Crawler, jobQueue *JobQueue) Worker {
......@@ -78,7 +77,7 @@ func (w Worker) checkURL(entry *MediaEntry) error {
}
client := &http.Client{}
ctx, _ := context.WithTimeout(context.Background(), timeout)
ctx, _ := context.WithTimeout(context.Background(), w.cr.webTimeout)
req, err := http.NewRequestWithContext(ctx, "GET", url.String(), nil)
if err != nil {
return emperror.Wrapf(err, "cannot create GET request for %s", url.String())
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment