App.scala 2.75 KB
Newer Older
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
1
/*
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
2
 * Import Process Delete
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
3
 * Copyright (C) 2021 Memoriav
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
4
5
 *
 * This program is free software: you can redistribute it and/or modify
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
6
7
8
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
9
10
11
12
13
14
15
16
17
18
19
20
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */

package ch.memobase

21
import ch.memobase.models.{DeleteMessage, ParserIgnore, Report}
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
22
23
24
import org.apache.logging.log4j.scala.Logging

import scala.collection.immutable.HashSet
25
import scala.util.{Failure, Success}
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
26
27
28
29
30
31
32
33
34
35
36

object App
  extends KProducer
    with scala.App
    with ArgParser
    with AppSettings
    with KConsumer
    with MsgFilter
    with Logging {

  parse(args) match {
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
37
    case Some((sessionId, filters, dryRun)) =>
38
39
      var counter = 0
      var matchCounter = 0
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
40
41
42
      try {
        Stream
          .continually(poll)
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
43
          .flatMap(records => {
44
45
            logger.info(s"Processing records batch with size ${records.size}; ${counter} processed so far")
            counter += records.size
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
46
            records
47
48
49
              .flatMap { record =>
                Report(record) match {
                  case Success(rep) => Some(rep)
50
51
52
                  case Failure(_: ParserIgnore) =>
                    logger.debug("Ignoring irrelevant message")
                    None
53
54
                  case Failure(ex) =>
                    logger.warn(s"Ignoring message because parsing failed: ${ex.getMessage}")
55
                    logger.info(s"${record.value()}")
56
57
58
                    None
                }
              }
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
59
              .filter(record => filters.forall(f => f(record)))
60
61
62
63
64
              .map(record => {
                matchCounter += 1
                logger.info(s"${matchCounter} matches so far")
                record
              })
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
65
              .foldLeft(HashSet[DeleteMessage]())((agg, record) => {
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
66
                logger.info(s"Size of delete messages: ${agg.size}")
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
67
68
69
                agg + DeleteMessage(record, sessionId)
              }
              )
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
70
          }
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
71
          )
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
72
          .foreach(id => sendDelete(id, dryRun))
Sebastian Schüpbach's avatar
Sebastian Schüpbach committed
73
74
75
76
77
78
79
80
81
82
83
84
85
86
      } catch {
        case e: Exception =>
          logger.error(e)
          sys.exit(1)
      } finally {
        logger.info("Shutting down application")
        closeConsumer()
        closeProducer()
      }
    case None =>
      logger.error("Invalid command-line arguments!")
      sys.exit(1)
  }
}