App.scala 2.84 KB
Newer Older
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
1
/*
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
2
 * Import Process Delete
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
3
 * Copyright (C) 2021 Memoriav
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
4
5
 *
 * This program is free software: you can redistribute it and/or modify
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
6
7
8
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
9
10
11
12
13
14
15
16
17
18
19
20
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */

package ch.memobase

21
import ch.memobase.models.{DeleteMessage, ParserIgnore, Report}
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
22
import org.apache.kafka.clients.consumer.KafkaConsumer
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
23
24
import org.apache.logging.log4j.scala.Logging

Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
25
26
import java.time.Duration
import scala.collection.JavaConverters._
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
27
import scala.collection.mutable
28
import scala.util.{Failure, Success}
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
29
30
31
32
33
34
35
36
37
38
39

object App
  extends KProducer
    with scala.App
    with ArgParser
    with AppSettings
    with KConsumer
    with MsgFilter
    with Logging {

  parse(args) match {
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
40
    case Some((sessionId, filters, dryRun)) =>
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
41
      try {
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
42
43
        var continue = true
        var processed = 0
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
44
45
46
47
48
49
50
51
52
        val matches: mutable.HashSet[DeleteMessage] = mutable.HashSet()
        Stream.continually(poll).takeWhile(_.nonEmpty)
          .flatten
          .zip(Stream.from(1))
          .foreach(p => if (p._2 % 250000 == 0) {
            logger.info(s"${p._2} records processed. Looking at ${p._1.key()} right now")
          })
        /* while (continue) {
          val records = poll
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
53
54
55
          if (records.isEmpty) {
            continue = false
          }
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
56
          for (record <- records) {
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
57
            processed += 1
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
58
59
60
            Report(record) match {
              case Success(rep) =>
                if (filters.forall(f => f(rep))) {
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
61
                  matches += DeleteMessage(rep, sessionId)
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
62
63
64
65
66
67
68
                }
              case Failure(_: ParserIgnore) =>
                logger.debug("Ignoring irrelevant message")
              case Failure(ex) =>
                logger.warn(s"Ignoring message because parsing failed: ${ex.getMessage}")
                logger.info(s"${record.value()}")
            }
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
69
70
            if (processed % 250000 == 0) {
              logger.info(s"$processed records processed; ${matches.size} matches")
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
71
            }
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
72
          }
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
73
        }
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
74
75
76
        for (m <- matches) {
          sendDelete(m, dryRun)
        } */
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
77
78
79
80
81
82
83
84
85
86
87
88
89
90
      } catch {
        case e: Exception =>
          logger.error(e)
          sys.exit(1)
      } finally {
        logger.info("Shutting down application")
        closeConsumer()
        closeProducer()
      }
    case None =>
      logger.error("Invalid command-line arguments!")
      sys.exit(1)
  }
}