banner functionality removed & config cleaned

cmd/main/config.go

@@ -34,15 +34,6 @@ type Meta struct {
 	Cron     string
 }
 
-type Banner struct {
-	Timeout   duration
-	Workers   int
-	PageSize  int
-	Cron      string
-	Folder    string
-	Timestamp string
-}
-
 type Crawler struct {
 	OK         duration
 	Error      duration
@@ -64,13 +55,9 @@ type Config struct {
 	JwtAlg    []string
 	DB        CfgDBMySQL
 	TempDir   string
-	Siegfried string
-	Ffmpeg    string
-	Ffprobe   string
 	Indexer   string
 	Crawler   Crawler
 	Metadata  Meta
-	Banner    Banner
 	FileMap   []FileMap
 }
 

cmd/main/main.go

@@ -108,20 +108,12 @@ func main() {
 		config.Crawler.PageSize,
 		config.DB.Schema,
 		config.TempDir,
-		config.Siegfried,
 		config.Crawler.OK.Duration,
 		config.Crawler.Error.Duration,
 		config.Crawler.ErrorNew.Duration,
 		config.Metadata.Timeout.Duration,
 		config.Metadata.Workers,
 		config.Metadata.PageSize,
-		config.Banner.Timeout.Duration,
-		config.Banner.Workers,
-		config.Banner.PageSize,
-		config.Banner.Timestamp,
-		config.Banner.Folder,
-		config.Ffmpeg,
-		config.Ffprobe,
 		config.Indexer,
 		fm,
 		log)

configs/memocrawler.toml

@@ -7,10 +7,7 @@ keypem = "" # tls client key file in PEM format
 jwtkey = "swordfish"
 jwtalg = ["HS256", "HS384", "HS512"] # "hs256" "hs384" "hs512" "es256" "es384" "es512" "ps256" "ps384" "ps512"
 tempdir = "C:/temp/"
-siegfried = "http://localhost:5138/identify/[[PATH]]?format=json"
 indexer = "http://localhost:81"
-ffmpeg = "/usr/local/bin/ffmpeg2"
-ffprobe = "/usr/local/bin/ffprobe2"
 
 [crawler]
     headersize = 5000 # number of bytes which are send to siegfried
@@ -28,14 +25,6 @@ ffprobe = "/usr/local/bin/ffprobe2"
     pagesize = 400
     cron = "32 * * * *" # cron format (https://pkg.go.dev/github.com/robfig/cron?tab=doc)
 
-[banner]
-    workers = 2
-    folder = "c:/temp/banner"
-    timeout = "30s"
-    pagesize = 200
-    timestamp = "00:00:12"
-    cron = "32 * * * *" # cron format (https://pkg.go.dev/github.com/robfig/cron?tab=doc)
-
 [[filemap]]
 alias = "c"
 folder = "c:/"

pkg/memocrawler/crawler.go

@@ -27,20 +27,12 @@ type Crawler struct {
 	pageSize         int
 	schema           string
 	tempDir          string
-	siegfried        Siegfried
 	crawlOK          time.Duration
 	crawlError       time.Duration
 	crawlErrorNew    time.Duration
 	metaTimeout      time.Duration
 	metaWorkers      int
 	metaPageSize     int
-	bannerTimeout    time.Duration
-	bannerWorkers    int
-	bannerPageSize   int
-	bannerTimestamp  string
-	bannerFolder     string
-	ffmpeg           string
-	ffprobe          string
 	indexer          string
 	bannerfolder     string
 	bannertimeout    time.Duration
@@ -59,17 +51,12 @@ func NewCrawler(
 	workers int,
 	pageSize int,
 	schema string,
-	tempDir, siegfried string,
+	tempDir string,
 	crawlOK, crawlError, crawlErrorNew time.Duration,
 	metaTimeout time.Duration,
 	metaWorkers int,
 	metaPageSize int,
-	bannerTimeout time.Duration,
-	bannerWorkers int,
-	bannerPageSize int,
-	bannerTimestamp string,
-	bannerFolder string,
-	ffmpeg, ffprobe, indexer string,
+	indexer string,
 	mapping *memostream.FileMapper,
 	log *logging.Logger) *Crawler {
 	cr := &Crawler{
@@ -78,7 +65,6 @@ func NewCrawler(
 		pageSize:       pageSize,
 		schema:         schema,
 		tempDir:        tempDir,
-		siegfried:      Siegfried{surl: siegfried},
 		indexer:        indexer,
 		crawlOK:        crawlOK,
 		crawlError:     crawlError,
@@ -86,13 +72,6 @@ func NewCrawler(
 		metaTimeout:    metaTimeout,
 		metaWorkers:    metaWorkers,
 		metaPageSize:   metaPageSize,
-		bannerTimeout:  bannerTimeout,
-		bannerWorkers:  bannerWorkers,
-		bannerPageSize: bannerPageSize,
-		bannerTimestamp: bannerTimestamp,
-		bannerFolder:    bannerFolder,
-		ffmpeg:          ffmpeg,
-		ffprobe:         ffprobe,
 		mapping:         mapping,
 		log:             log,
 	}