App.scala

/*
 * Import Process Delete
 * Copyright (C) 2021 Memoriav
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <https://www.gnu.org/licenses/>.
 */

package ch.memobase

import ch.memobase.models.{DeleteMessage, ParserIgnore, Report}
import org.apache.kafka.clients.consumer.KafkaConsumer
import org.apache.logging.log4j.scala.Logging

import java.time.Duration
import scala.collection.JavaConverters._
import scala.util.{Failure, Success}

object App
  extends KProducer
    with scala.App
    with ArgParser
    with AppSettings
    with KConsumer
    with MsgFilter
    with Logging {

  lazy private val consumer = {
    val consumer = new KafkaConsumer[String, String](consumerProps)
    logger.debug(s"Subscribing to topic $inputTopic")
    consumer.subscribe(List(inputTopic).asJava)
    consumer
  }

  parse(args) match {
    case Some((sessionId, filters, dryRun)) =>
      try {
        var continue = true
        var processed = 0
        var matches = 0
        while (continue) {
          val records = consumer.poll(Duration.ofMillis(pollTimeout)).asScala
          if (records.isEmpty) {
            continue = false
          }
          for (record <- records) {
            processed += 1
            Report(record) match {
              case Success(rep) =>
                if (filters.forall(f => f(rep))) {
                  matches += 1
                  sendDelete(DeleteMessage(rep, sessionId), dryRun)
                }
              case Failure(_: ParserIgnore) =>
                logger.debug("Ignoring irrelevant message")
              case Failure(ex) =>
                logger.warn(s"Ignoring message because parsing failed: ${ex.getMessage}")
                logger.info(s"${record.value()}")
            }
            if (processed % 100000 == 0) {
              logger.info(s"$processed records processed; $matches matches")
            }
          }
        }
      } catch {
        case e: Exception =>
          logger.error(e)
          sys.exit(1)
      } finally {
        logger.info("Shutting down application")
        closeConsumer()
        closeProducer()
      }
    case None =>
      logger.error("Invalid command-line arguments!")
      sys.exit(1)
  }
}