Verified Commit 0bb33770 authored by Sebastian Schüpbach's avatar Sebastian Schüpbach

Merge branch 'master' of gitlab.switch.ch:memoriav/memobase-2020/services/url-checker

parents 965d8690 b1fb6203
......@@ -34,15 +34,6 @@ type Meta struct {
Cron string
}
type Banner struct {
Timeout duration
Workers int
PageSize int
Cron string
Folder string
Timestamp string
}
type Crawler struct {
OK duration
Error duration
......@@ -64,13 +55,9 @@ type Config struct {
JwtAlg []string
DB CfgDBMySQL
TempDir string
Siegfried string
Ffmpeg string
Ffprobe string
Indexer string
Crawler Crawler
Metadata Meta
Banner Banner
FileMap []FileMap
}
......
......@@ -108,20 +108,12 @@ func main() {
config.Crawler.PageSize,
config.DB.Schema,
config.TempDir,
config.Siegfried,
config.Crawler.OK.Duration,
config.Crawler.Error.Duration,
config.Crawler.ErrorNew.Duration,
config.Metadata.Timeout.Duration,
config.Metadata.Workers,
config.Metadata.PageSize,
config.Banner.Timeout.Duration,
config.Banner.Workers,
config.Banner.PageSize,
config.Banner.Timestamp,
config.Banner.Folder,
config.Ffmpeg,
config.Ffprobe,
config.Indexer,
fm,
log)
......
......@@ -7,10 +7,7 @@ keypem = "" # tls client key file in PEM format
jwtkey = "swordfish"
jwtalg = ["HS256", "HS384", "HS512"] # "hs256" "hs384" "hs512" "es256" "es384" "es512" "ps256" "ps384" "ps512"
tempdir = "C:/temp/"
siegfried = "http://localhost:5138/identify/[[PATH]]?format=json"
indexer = "http://localhost:81"
ffmpeg = "/usr/local/bin/ffmpeg2"
ffprobe = "/usr/local/bin/ffprobe2"
[crawler]
headersize = 5000 # number of bytes which are send to siegfried
......@@ -28,14 +25,6 @@ ffprobe = "/usr/local/bin/ffprobe2"
pagesize = 400
cron = "32 * * * *" # cron format (https://pkg.go.dev/github.com/robfig/cron?tab=doc)
[banner]
workers = 2
folder = "c:/temp/banner"
timeout = "30s"
pagesize = 200
timestamp = "00:00:12"
cron = "32 * * * *" # cron format (https://pkg.go.dev/github.com/robfig/cron?tab=doc)
[[filemap]]
alias = "c"
folder = "c:/"
......
......@@ -27,20 +27,12 @@ type Crawler struct {
pageSize int
schema string
tempDir string
siegfried Siegfried
crawlOK time.Duration
crawlError time.Duration
crawlErrorNew time.Duration
metaTimeout time.Duration
metaWorkers int
metaPageSize int
bannerTimeout time.Duration
bannerWorkers int
bannerPageSize int
bannerTimestamp string
bannerFolder string
ffmpeg string
ffprobe string
indexer string
bannerfolder string
bannertimeout time.Duration
......@@ -59,17 +51,12 @@ func NewCrawler(
workers int,
pageSize int,
schema string,
tempDir, siegfried string,
tempDir string,
crawlOK, crawlError, crawlErrorNew time.Duration,
metaTimeout time.Duration,
metaWorkers int,
metaPageSize int,
bannerTimeout time.Duration,
bannerWorkers int,
bannerPageSize int,
bannerTimestamp string,
bannerFolder string,
ffmpeg, ffprobe, indexer string,
indexer string,
mapping *memostream.FileMapper,
log *logging.Logger) *Crawler {
cr := &Crawler{
......@@ -78,7 +65,6 @@ func NewCrawler(
pageSize: pageSize,
schema: schema,
tempDir: tempDir,
siegfried: Siegfried{surl: siegfried},
indexer: indexer,
crawlOK: crawlOK,
crawlError: crawlError,
......@@ -86,13 +72,6 @@ func NewCrawler(
metaTimeout: metaTimeout,
metaWorkers: metaWorkers,
metaPageSize: metaPageSize,
bannerTimeout: bannerTimeout,
bannerWorkers: bannerWorkers,
bannerPageSize: bannerPageSize,
bannerTimestamp: bannerTimestamp,
bannerFolder: bannerFolder,
ffmpeg: ffmpeg,
ffprobe: ffprobe,
mapping: mapping,
log: log,
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment