Unverified Commit f37d4af2 authored by Sebastian Schüpbach's avatar Sebastian Schüpbach
Browse files

first commit

parents
Pipeline #22743 passed with stages
in 6 minutes and 17 seconds
.bloop/
.idea/
.metals/
target/
stages:
- test
- build
- publish
include:
- project: 'memoriav/memobase-2020/utilities/ci-templates'
ref: modular
file: 'helm/default.yml'
- project: 'memoriav/memobase-2020/utilities/ci-templates'
ref: modular
file: 'sbt/default.yml'
- project: 'memoriav/memobase-2020/utilities/ci-templates'
ref: modular
file: 'docker/default.yml'
\ No newline at end of file
version = "2.7.4"
FROM openjdk:8-jre-alpine
RUN apk add --no-cache ffmpeg
ADD target/scala-2.12/app.jar /app/app.jar
CMD java -jar /app/app.jar
This diff is collapsed.
# Reports Aggregation
\ No newline at end of file
import Dependencies._
ThisBuild / scalaVersion := "2.12.13"
ThisBuild / version := "0.1.0-SNAPSHOT"
ThisBuild / organization := "ch.memobase"
ThisBuild / organizationName := "Memoriav"
ThisBuild / git.gitTagToVersionNumber := { tag: String =>
if (tag matches "[0-9]+\\..*") {
Some(tag)
} else {
None
}
}
lazy val root = (project in file("."))
.enablePlugins(GitVersioning)
.settings(
name := "Reports Aggregation",
assemblyJarName in assembly := "app.jar",
test in assembly := {},
assemblyMergeStrategy in assembly := {
case "log4j.properties" => MergeStrategy.first
case "log4j2.xml" => MergeStrategy.first
case "module-info.class" => MergeStrategy.discard
case x =>
val oldStrategy = (assemblyMergeStrategy in assembly).value
oldStrategy(x)
},
mainClass in assembly := Some("ch.memobase.App"),
libraryDependencies ++= Seq(
kafkaStreams,
log4jApi,
log4jCore,
log4jSlf4j,
log4jScala,
scalaUri,
upickle,
kafkaStreamsTestUtils % Test,
scalaTest % Test
)
)
apiVersion: v2
name: media-metadata-preprocessor
description: Preprocesses records for a potential enrichment with media metadata
type: application
version: 0.0.0
appVersion: 0.0.0
maintainers:
- name: Sebastian Schüpbach
email: sebastian.schuepbach@unibas.ch
deploymentName: ip-media-metadata-preprocessor
inputTopicName: import-process-media-metadata
outputTopicAVName: import-process-av-enrichment
outputTopicIgnoreName: import-process-normalization
outputTopicImageName: import-process-image-enrichment
reportingTopicName: import-process-reporting
\ No newline at end of file
apiVersion: v1
kind: ConfigMap
metadata:
name: "{{ .Values.deploymentName }}-config"
namespace: memobase
data:
APPLICATION_ID: "{{ .Values.applicationId }}"
TOPIC_IN: "{{ .Values.inputTopicName }}"
TOPIC_OUT_AV: "{{ .Values.outputTopicAVName }}"
TOPIC_OUT_IMAGE: "{{ .Values.outputTopicImageName }}"
TOPIC_OUT_IGNORE: "{{ .Values.outputTopicIgnoreName }}"
TOPIC_PROCESS: "{{ .Values.reportingTopicName }}"
apiVersion: apps/v1
kind: Deployment
metadata:
name: "{{ .Values.deploymentName }}"
namespace: memobase
labels:
jobType: "import-process-deployment"
spec:
selector:
matchLabels:
app: {{ .Values.deploymentName }}
replicas: 1
template:
metadata:
labels:
app: {{ .Values.deploymentName }}
tier: import-process
spec:
restartPolicy: Always
containers:
- name: "{{ .Values.deploymentName }}-container"
image: "{{ .Values.registry }}/{{ .Values.image }}:{{ .Values.tag }}"
imagePullPolicy: Always
command: [ "java" ]
args: [ "-Xms64m", "-Xmx256g", "-jar", "/app/app.jar" ]
resources:
requests:
cpu: "0.2"
memory: "128Mi"
limits:
cpu: "0.5"
memory: "300Mi"
envFrom:
- configMapRef:
name: "{{ .Values.bootstrapServers }}"
- configMapRef:
name: "{{ .Values.deploymentName }}-config"
############################################
## Values in this section are the same for #
## all jobs #
############################################
#image values
registry: "cr.gitlab.switch.ch"
image: "memoriav/memobase-2020/services/import-process/media-metadata-preprocessor"
tag: "latest"
deploymentName: ip-media-metadata-preprocessor
applicationId: media-metadata-preprocessor-app
bootstrapServers: prod-kafka-bootstrap-servers
inputTopicName: import-process-media-metadata
outputTopicAVName: import-process-av-enrichment
outputTopicIgnoreName: import-process-normalization
outputTopicImageName: import-process-image-enrichment
reportingTopicName: import-process-reporting
/*
* Reports Aggregation
* Copyright (C) 2021 Memoriav
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
import sbt._
object Dependencies {
lazy val kafkaV = "2.3.1"
lazy val log4jV = "2.11.2"
lazy val scalatestV = "3.1.2"
lazy val kafkaStreams = "org.apache.kafka" %% "kafka-streams-scala" % kafkaV
lazy val kafkaStreamsTestUtils =
"org.apache.kafka" % "kafka-streams-test-utils" % kafkaV
lazy val log4jApi = "org.apache.logging.log4j" % "log4j-api" % log4jV
lazy val log4jCore = "org.apache.logging.log4j" % "log4j-core" % log4jV
lazy val log4jScala = "org.apache.logging.log4j" %% "log4j-api-scala" % "11.0"
lazy val log4jSlf4j = "org.apache.logging.log4j" % "log4j-slf4j-impl" % log4jV
lazy val scalaTest = "org.scalatest" %% "scalatest" % scalatestV
lazy val scalaUri = "io.lemonlabs" %% "scala-uri" % "2.2.3"
lazy val upickle = "com.lihaoyi" %% "upickle" % "1.2.3"
}
#
# Reports Aggregation
# Copyright (C) 2021 Memoriav
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
# published by the Free Software Foundation, either version 3 of the
# License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
#
sbt.version=1.3.10
// DO NOT EDIT! This file is auto-generated.
// This file enables sbt-bloop to create bloop config files.
addSbtPlugin("ch.epfl.scala" % "sbt-bloop" % "1.4.8")
addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.10")
addSbtPlugin("com.typesafe.sbt" % "sbt-git" % "1.0.0")
addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "1.0.0")
\ No newline at end of file
// DO NOT EDIT! This file is auto-generated.
// This file enables sbt-bloop to create bloop config files.
addSbtPlugin("ch.epfl.scala" % "sbt-bloop" % "1.4.8")
// DO NOT EDIT! This file is auto-generated.
// This file enables sbt-bloop to create bloop config files.
addSbtPlugin("ch.epfl.scala" % "sbt-bloop" % "1.4.8")
<!--
~ Reports Aggregation
~ Copyright (C) 2021 Memoriav
~
~ This program is free software: you can redistribute it and/or modify
~ it under the terms of the GNU Affero General Public License as
~ published by the Free Software Foundation, either version 3 of the
~ License, or (at your option) any later version.
~
~ This program is distributed in the hope that it will be useful,
~ but WITHOUT ANY WARRANTY; without even the implied warranty of
~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
~ GNU Affero General Public License for more details.
~
~ You should have received a copy of the GNU Affero General Public License
~ along with this program. If not, see <https://www.gnu.org/licenses/>.
~
-->
<scalastyle>
<name>Scalastyle standard configuration</name>
<check level="warning" class="org.scalastyle.file.FileTabChecker" enabled="true"/>
<check level="warning" class="org.scalastyle.file.FileLengthChecker" enabled="true">
<parameters>
<parameter name="maxFileLength"><![CDATA[800]]></parameter>
</parameters>
</check>
<check level="warning" class="org.scalastyle.file.HeaderMatchesChecker" enabled="true">
<parameters>
<parameter name="header"><![CDATA[/*
* Reports Aggregation
* Copyright (C) 2021 Memoriav
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.]]></parameter>
</parameters>
</check>
<check level="warning" class="org.scalastyle.scalariform.SpacesAfterPlusChecker" enabled="true"/>
<check level="warning" class="org.scalastyle.file.WhitespaceEndOfLineChecker" enabled="true"/>
<check level="warning" class="org.scalastyle.scalariform.SpacesBeforePlusChecker" enabled="true"/>
<check level="warning" class="org.scalastyle.file.FileLineLengthChecker" enabled="true">
<parameters>
<parameter name="maxLineLength"><![CDATA[160]]></parameter>
<parameter name="tabSize"><![CDATA[4]]></parameter>
</parameters>
</check>
<check level="warning" class="org.scalastyle.scalariform.ClassNamesChecker" enabled="true">
<parameters>
<parameter name="regex"><![CDATA[[A-Z][A-Za-z]*]]></parameter>
</parameters>
</check>
<check level="warning" class="org.scalastyle.scalariform.ObjectNamesChecker" enabled="true">
<parameters>
<parameter name="regex"><![CDATA[[A-Z][A-Za-z]*]]></parameter>
</parameters>
</check>
<check level="warning" class="org.scalastyle.scalariform.PackageObjectNamesChecker" enabled="true">
<parameters>
<parameter name="regex"><![CDATA[^[a-z][A-Za-z]*$]]></parameter>
</parameters>
</check>
<check level="warning" class="org.scalastyle.scalariform.EqualsHashCodeChecker" enabled="true"/>
<check level="warning" class="org.scalastyle.scalariform.IllegalImportsChecker" enabled="true">
<parameters>
<parameter name="illegalImports"><![CDATA[sun._,java.awt._]]></parameter>
</parameters>
</check>
<check level="warning" class="org.scalastyle.scalariform.ParameterNumberChecker" enabled="true">
<parameters>
<parameter name="maxParameters"><![CDATA[8]]></parameter>
</parameters>
</check>
<check level="warning" class="org.scalastyle.scalariform.MagicNumberChecker" enabled="true">
<parameters>
<parameter name="ignore"><![CDATA[-1,0,1,2,3]]></parameter>
</parameters>
</check>
<check level="warning" class="org.scalastyle.scalariform.NoWhitespaceBeforeLeftBracketChecker" enabled="true"/>
<check level="warning" class="org.scalastyle.scalariform.NoWhitespaceAfterLeftBracketChecker" enabled="true"/>
<check level="warning" class="org.scalastyle.scalariform.ReturnChecker" enabled="true"/>
<check level="warning" class="org.scalastyle.scalariform.NullChecker" enabled="true"/>
<check level="warning" class="org.scalastyle.scalariform.NoCloneChecker" enabled="true"/>
<check level="warning" class="org.scalastyle.scalariform.NoFinalizeChecker" enabled="true"/>
<check level="warning" class="org.scalastyle.scalariform.CovariantEqualsChecker" enabled="true"/>
<check level="warning" class="org.scalastyle.scalariform.StructuralTypeChecker" enabled="true"/>
<check level="warning" class="org.scalastyle.file.RegexChecker" enabled="true">
<parameters>
<parameter name="regex"><![CDATA[println]]></parameter>
</parameters>
</check>
<check level="warning" class="org.scalastyle.scalariform.NumberOfTypesChecker" enabled="true">
<parameters>
<parameter name="maxTypes"><![CDATA[30]]></parameter>
</parameters>
</check>
<check level="warning" class="org.scalastyle.scalariform.CyclomaticComplexityChecker" enabled="true">
<parameters>
<parameter name="maximum"><![CDATA[10]]></parameter>
</parameters>
</check>
<check level="warning" class="org.scalastyle.scalariform.UppercaseLChecker" enabled="true"/>
<check level="warning" class="org.scalastyle.scalariform.SimplifyBooleanExpressionChecker" enabled="true"/>
<check level="warning" class="org.scalastyle.scalariform.IfBraceChecker" enabled="true">
<parameters>
<parameter name="singleLineAllowed"><![CDATA[true]]></parameter>
<parameter name="doubleLineAllowed"><![CDATA[false]]></parameter>
</parameters>
</check>
<check level="warning" class="org.scalastyle.scalariform.MethodLengthChecker" enabled="true">
<parameters>
<parameter name="maxLength"><![CDATA[50]]></parameter>
</parameters>
</check>
<check level="warning" class="org.scalastyle.scalariform.MethodNamesChecker" enabled="true">
<parameters>
<parameter name="regex"><![CDATA[^[a-z][A-Za-z0-9]*$]]></parameter>
</parameters>
</check>
<check level="warning" class="org.scalastyle.scalariform.NumberOfMethodsInTypeChecker" enabled="true">
<parameters>
<parameter name="maxMethods"><![CDATA[30]]></parameter>
</parameters>
</check>
<check level="warning" class="org.scalastyle.scalariform.PublicMethodsHaveTypeChecker" enabled="true"/>
<check level="warning" class="org.scalastyle.file.NewLineAtEofChecker" enabled="true"/>
<check level="warning" class="org.scalastyle.file.NoNewLineAtEofChecker" enabled="false"/>
</scalastyle>
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Reports Aggregation
~ Copyright (C) 2021 Memoriav
~
~ This program is free software: you can redistribute it and/or modify
~ it under the terms of the GNU Affero General Public License as
~ published by the Free Software Foundation, either version 3 of the
~ License, or (at your option) any later version.
~
~ This program is distributed in the hope that it will be useful,
~ but WITHOUT ANY WARRANTY; without even the implied warranty of
~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
~ GNU Affero General Public License for more details.
~
~ You should have received a copy of the GNU Affero General Public License
~ along with this program. If not, see <https://www.gnu.org/licenses/>.
~
-->
<Configuration status="info" name="reports-aggregation" packages="">
<Appenders>
<Console name="STDOUT" target="SYSTEM_OUT">
<PatternLayout pattern="[%-5p] [%d{yyyy-MM-dd HH:mm:ss}] [%c{1.}:%L] %m%n"/>
</Console>
</Appenders>
<Loggers>
<Root level="info">
<AppenderRef ref="STDOUT"/>
</Root>
</Loggers>
</Configuration>
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment