Merge branch 'master' of...

Merge branch 'master' of gitlab.switch.ch:memoriav/memobase-2020/services/postprocessing/media-converter

.gitlab-ci.yml

@@ -24,7 +24,7 @@ cache:
 test-sbt:
   extends: .test-sbt
   before_script:
-    - apt-get update && apt-get install -y ffmpeg
+    - apt-get update --allow-releaseinfo-change && apt-get install -y ffmpeg imagemagick
 assembly-sbt:
   extends: .assembly-sbt
 
@@ -35,4 +35,4 @@ build-latest-image:
   extends: .build-latest-image
 
 build-feature-branch-image:
-  extends: .build-feature-branch-image
\ No newline at end of file
+  extends: .build-feature-branch-image

.scalafmt.conf

+version = "3.0.5"
+runner.dialect = scala213
\ No newline at end of file

Dockerfile

 FROM openjdk:8-jre
 RUN apt-get update && \
-apt-get install -y ffmpeg && \
+apt-get install -y ffmpeg imagemagick && \
 apt-get autoremove -y && \
 apt-get clean
 ADD target/scala-2.12/app.jar /app/app.jar

README.md

 # Media Converter
 
-The __Media Converter__ is an intermediary service which takes care of media file postprocessing
- upon receiving notifications on changes in the Fedora repository. More precisely, the service 
- 1. extracts the path to the media file from the Fedora API based on the received ID
- 2. transcodes this file if required (e.g. jpg -> jp2),
- 3. copies the file to a designated directory accessible by the mediaserver or the Cantaloupe IIIF imageserver and
- 4. updates the Mediaserver database with the path to the copied file.
- 
-## Components
-
-The service is packaged as a Docker image. It comprises the following components:
-
-- The main application, responsible for communicating with [Kafka](https://kafka.apache.org), 
-[Fedora](https://duraspace.org/fedora/) and the 
-[Mediaserver](https://gitlab.switch.ch/memoriav/memobase-2020/services/streaming-server) DB as well as handling the 
-media files
-- ffmpeg for transcoding video and audio
-- imagemagick for transcoding images
-
-For a comprehensive list consult the [Dockerfile](./Dockerfile).
- 
+The __Media Converter__ is responsible for preparing media files for consumption by end users. This comprises:
+
+* Copying files from the source folder (the sFTP directory) to a dedicated media directory directly accessible by the
+  media file providers like
+  the [media server](https://gitlab.switch.ch/memoriav/memobase-2020/services/streaming-server) or
+  the [IIIF image server](https://gitlab.switch.ch/memoriav/memobase-2020/services/cantaloupe-docker). Besides the
+  distribution copies these can also be preview images for videos ("thumbnails")
+* In the case of audio files repackaging the content in an MPEG4 container
+* Creating small "snippets" from the audio file which are in turn used by the frontend to create sonograms
+
+## Copying
+
+The service gets the needed media files via
+the [Media File Distributor](https://gitlab.switch.ch/memoriav/memobase-2020/services/import-process/media-distributor-service)
+, which in turn directly reads from the collections directory on the sFTP server. The fetched files are written to the
+respective media file directory. In the case of the Memobase workflow these directories are directly mounted in the
+service containers which need them.
+
+## Conversions
+
+* Audio files: Repackages files in an mpeg4 container with the help of `ffmpeg` and sets the moov atom at the beginning
+  of the file (`-movflags faststart`)
+* Image files: Copies files as-is. An additional thumbnail is created.
+* Video files: Copies files as-is
+
+## Creating snippets
+
+In order to provide content for sonograms and a teaser on the frontend, small snippets of the first x seconds from the
+audio files are produced. A relatively small snippet size helps to avoid getting only a solid black bar as a sonogram (
+which would be the case if one compresses a sonogram of a lengthy audio track to a width which fits the icon size used
+in the frontend).
+
 ## Configuration
 
-In order to configure the application, you can either modify the commented 
-[example configuration file](./src/main/resources/app.yml) and deploy it as a configuration in your cluster or set the
-respective environment variables. For a comprehensive list of possible variables for the Kafka producer settings see 
-[here](https://gitlab.switch.ch/memoriav/memobase-2020/libraries/service-utilities/-/blob/master/src/main/kotlin/settings/KafkaSettings.kt#L124).
\ No newline at end of file
+In order to work as expected, the service needs to have a couple of environment variables set:
+
+* `KAFKA_BOOTSTRAP_SERVERS`: Comma-separated list of Kafka bootstrap server addresses
+* `TOPIC_IN`: Kafka topic where incoming topics are read from
+* `TOPIC_PROCESS`: Kafka topic where status reports are written to
+* `CLIENT_ID`: Kafka client id
+* `GROUP_ID`: Kafka consumer group id
+* `AUDIO_SNIPPET_DURATION`: Number of seconds which are taken from the beginning of the audio track to produce the
+  snippet
+* `EXTERNAL_BASE_URL`: Base URL under which the resource is available
+* `MEDIA_FOLDER_ROOT_PATH`: Path to the mounted media folder (i.e. the folder where the media files are copied to)
+* `THUMBNAIL_FOLDER_PATH`: Path to the thumbnail folder
+* `THUMBNAIL_WIDTH`: Optional width of produced thumbnails
+* `THUMBNAIL_HEIGHT`: Optional height of produced thumbnails
+* `DISTRIBUTOR_URL`: Address of the respective Media File Distributor instance
+* `CONNECTION_RETRY_AFTER_MS`: Delay in milliseconds after which a reconnection to the Media File Distributor takes
+  place
+* `CONNECTION_MAX_RETRIES`: Maximum number of connection retries
\ No newline at end of file

helm-charts/helm-values/di-media-converter-prod.yaml

@@ -5,6 +5,8 @@ k8sRequestsMemory: "1Gi"
 k8sLimitsCpu: "1"
 k8sLimitsMemory: "11Gi"
 
+debugLevel: "warn"
+
 kafkaConfigs: prod-kafka-bootstrap-servers
 inputTopicName: mb-di-processed-records-prod
 reportingTopicName: mb-di-reporting-prod
@@ -17,6 +19,9 @@ externalBaseUrl: "https://memobase.ch/"
 distributorUrl: "http://mb-wf2.memobase.unibas.ch:3000"
 
 mediaFolderRootPath: "/data"
+thumbnailFolderPath: "/data/cached"
+thumbnailWidth: "640"
+thumbnailHeight: "0"
 mediaVolumeClaimName: media-volume-claim
 
 connectionRetryAfterMs: "10000"

helm-charts/helm-values/di-media-converter-stage.yaml

@@ -5,6 +5,8 @@ k8sRequestsMemory: "1Gi"
 k8sLimitsCpu: "1"
 k8sLimitsMemory: "11Gi"
 
+debugLevel: "warn"
+
 kafkaConfigs: prod-kafka-bootstrap-servers
 inputTopicName: mb-di-processed-records-stage
 reportingTopicName: mb-di-reporting-stage
@@ -14,9 +16,12 @@ clientId: stage-media-converter-client
 audioSnippetDuration: "30"
 
 externalBaseUrl: "https://memobase.ch/"
-distributorUrl: "http://mb-wf2.memobase.unibas.ch:3000"
+distributorUrl: "http://mb-wf2.memobase.unibas.ch:3001"
 
 mediaFolderRootPath: "/data"
+thumbnailFolderPath: "/data/cached"
+thumbnailWidth: "640"
+thumbnailHeight: "0"
 mediaVolumeClaimName: stage-media-volume-claim
 
 connectionRetryAfterMs: "10000"

helm-charts/helm-values/di-media-converter-test.yaml

@@ -3,7 +3,9 @@ k8sReplicas: 1
 k8sRequestsCpu: "0.2"
 k8sRequestsMemory: "1Gi"
 k8sLimitsCpu: "1"
-k8sLimitsMemory: "2Gi"
+k8sLimitsMemory: "4Gi"
+
+debugLevel: "debug"
 
 kafkaConfigs: test-kafka-bootstrap-servers
 inputTopicName: mb-di-processed-records-prod
@@ -14,9 +16,12 @@ clientId: test-media-converter-client
 audioSnippetDuration: "30"
 
 externalBaseUrl: "https://memobase.ch/"
-distributorUrl: "http://mb-wf2.memobase.unibas.ch:3000"
+distributorUrl: "http://mb-wf2.memobase.unibas.ch:3002"
 
 mediaFolderRootPath: "/data"
+thumbnailFolderPath: "/data/cached"
+thumbnailWidth: "640"
+thumbnailHeight: "0"
 mediaVolumeClaimName: test-media-volume-claim
 
 connectionRetryAfterMs: "10000"

helm-charts/templates/app-config.yaml

@@ -9,6 +9,7 @@ metadata:
     group: "{{ .Values.k8sGroupName }}"
 data:
   APPLICATION_ID: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}"
+  DEBUG_LEVEL: "{{ .Values.debugLevel }}"
   TOPIC_IN: "{{ .Values.inputTopicName }}"
   TOPIC_PROCESS: "{{ .Values.reportingTopicName }}"
   CLIENT_ID: "{{ .Values.clientId }}"
@@ -16,6 +17,9 @@ data:
   AUDIO_SNIPPET_DURATION: "{{ .Values.audioSnippetDuration }}"
   EXTERNAL_BASE_URL: "{{ .Values.externalBaseUrl }}"
   MEDIA_FOLDER_ROOT_PATH: "{{ .Values.mediaFolderRootPath }}"
+  THUMBNAIL_FOLDER_PATH: "{{ .Values.thumbnailFolderPath }}"
+  THUMBNAIL_WIDTH: "{{ .Values.thumbnailWidth }}"
+  THUMBNAIL_HEIGHT: "{{ .Values.thumbnailHeight }}"
   DISTRIBUTOR_URL: "{{ .Values.distributorUrl }}"
   CONNECTION_RETRY_AFTER_MS: "{{ .Values.connectionRetryAfterMs }}"
-  CONNECTION_MAX_RETRIES: "{{ .Values.connectionMaxRetries }}"
\ No newline at end of file
+  CONNECTION_MAX_RETRIES: "{{ .Values.connectionMaxRetries }}"

helm-charts/templates/deployment.yaml

@@ -5,21 +5,21 @@ metadata:
   namespace: "{{ .Values.k8sNamespace }}"
   labels:
     app: "{{ .Values.k8sName }}"
-    environment: "{{ .Values.k8sEnvironment }}"
     group: "{{ .Values.k8sGroupName }}"
+    environment: "{{ .Values.k8sEnvironment }}"
 spec:
   selector:
     matchLabels:
       app: "{{ .Values.k8sName }}"
-      environment: "{{ .Values.k8sEnvironment }}"
       group: "{{ .Values.k8sGroupName }}"
-  replicas: {{ .Values.k8sReplicas }}
+      environment: "{{ .Values.k8sEnvironment }}"
+  replicas: {{ .Values.k8sReplicas}}
   template:
     metadata:
       labels:
         app: "{{ .Values.k8sName }}"
-        environment: "{{ .Values.k8sEnvironment }}"
         group: "{{ .Values.k8sGroupName }}"
+        environment: "{{ .Values.k8sEnvironment }}"
         logging: "{{ .Values.k8sLogging }}"
     spec:
       containers:

helm-charts/values.yaml

@@ -6,6 +6,7 @@ k8sName: media-converter
 k8sNamespace: memobase
 k8sGroupId: di
 k8sGroupName: documents-import
+k8sLogging: scala
 k8sEnvironment: placeholder
 k8sLogging: sca
 k8sReplicas: placeholder
@@ -14,6 +15,8 @@ k8sRequestsMemory: placeholder
 k8sLimitsCpu: placeholder
 k8sLimitsMemory: placeholder
 
+debugLevel: placeholder
+
 kafkaConfigs: placeholder
 inputTopicName: placeholder
 reportingTopicName: placeholder
@@ -26,7 +29,10 @@ externalBaseUrl: placeholder
 distributorUrl: placeholder
 
 mediaFolderRootPath: placeholder
+thumbnailFolderPath: placeholder
+thumbnailWidth: placeholder
+thumbnailHeight: placeholder
 mediaVolumeClaimName: placeholder
 
 connectionRetryAfterMs: placeholder
-connectionMaxRetries: placeholder
\ No newline at end of file
+connectionMaxRetries: placeholder

project/Dependencies.scala

@@ -21,7 +21,7 @@ import sbt._
 
 object Dependencies {
   lazy val kafkaV = "2.7.0"
-  lazy val log4jV = "2.11.2"
+  lazy val log4jV = "2.17.0"
   lazy val scalatestV = "3.1.2"
 
   //lazy val fedoraClient = "org.memobase" % "fedora-client" % "0.6.1"
@@ -30,7 +30,7 @@ object Dependencies {
   lazy val log4jCore = "org.apache.logging.log4j" % "log4j-core" % log4jV
   lazy val log4jScala = "org.apache.logging.log4j" %% "log4j-api-scala" % "11.0"
   lazy val log4jSlf4j = "org.apache.logging.log4j" % "log4j-slf4j-impl" % log4jV
-  lazy val memobaseServiceUtils = "org.memobase" % "memobase-service-utilities" % "3.0.1"
+  lazy val memobaseServiceUtils = "org.memobase" % "memobase-service-utilities" % "3.1.2"
   lazy val scalaMock = "org.scalamock" %% "scalamock" % "5.0.0"
   lazy val scalatic = "org.scalactic" %% "scalactic" % scalatestV
   lazy val scalaTest = "org.scalatest" %% "scalatest" % scalatestV

project/metals.sbt

 // DO NOT EDIT! This file is auto-generated.
+
 // This file enables sbt-bloop to create bloop config files.
 
-addSbtPlugin("ch.epfl.scala" % "sbt-bloop" % "1.4.3-23-550c6c0a")
+addSbtPlugin("ch.epfl.scala" % "sbt-bloop" % "1.4.12")
+

src/main/resources/app.yml

@@ -2,6 +2,9 @@ app:
   audioSnippetDuration: ${AUDIO_SNIPPET_DURATION:?system}
   externalBaseUrl: ${EXTERNAL_BASE_URL:?system}
   mediaFolderRootPath: ${MEDIA_FOLDER_ROOT_PATH:?system}
+  thumbnailFolderPath: ${THUMBNAIL_FOLDER_PATH:?system}
+  thumbnailWidth: ${THUMBNAIL_WIDTH:?system}
+  thumbnailHeight: ${THUMBNAIL_HEIGHT:?system}
   distributorUrl: ${DISTRIBUTOR_URL:?system}
   connectionMaxRetries: ${CONNECTION_MAX_RETRIES:?system}
   connectionRetryAfterMs: ${CONNECTION_RETRY_AFTER_MS:?system}

src/main/resources/log4j2.xml

@@ -24,8 +24,11 @@
         </Console>
     </Appenders>
     <Loggers>
-        <Root level="info">
+        <Root level="${env:DEBUG_LEVEL:-warn}">
             <AppenderRef ref="STDOUT"/>
         </Root>
+        <Logger name="org.apache.kafka" level="warn">
+            <AppenderRef ref="STDOUT"/>
+        </Logger>
     </Loggers>
 </Configuration>
\ No newline at end of file

src/main/scala/ch/memobase/App.scala

@@ -33,6 +33,9 @@ object App extends scala.App with Logging with RecordUtils {
     "audioSnippetDuration",
     "externalBaseUrl",
     "mediaFolderRootPath",
+    "thumbnailFolderPath",
+    "thumbnailWidth",
+    "thumbnailHeight",
     "distributorUrl",
     "connectionMaxRetries",
     "connectionRetryAfterMs"

src/main/scala/ch/memobase/DisseminationCopyHandler.scala

@@ -97,9 +97,25 @@ class DisseminationCopyHandler(audioSnippetDuration: Int) extends Logging {
    */
   def createImageCopy(data: ByteArrayOutputStream, destFile: String, sourceFileType: MimeType): Try[Boolean] = Try {
     val destFileAsPath = Paths.get(destFile)
-    val copyRemoved = removeExistingFile(destFileAsPath)
+    val fileRemoved = removeExistingFile(destFileAsPath)
     writeData(data, destFileAsPath)
-    copyRemoved
+    fileRemoved
+  }
+
+  /**
+   * Creates a thumbnail representation of the image
+   *
+   * @param origFile Path to the dissemination copy of the image file
+   * @param destFile Path to the thumbnail representation of the image
+   * @param width    Optional width of thumbnail
+   * @param height   Optional height of thumbnail
+   * @return true if thumbnail was overwritten, false otherwise
+   */
+  def createImageThumbnail(origFile: String, destFile: String, width: Option[Int], height: Option[Int]): Try[Boolean] = Try {
+    val destFileAsPath = Paths.get(destFile)
+    val fileRemoved = removeExistingFile(destFileAsPath)
+    MediaTransformations.createImageThumbnail(origFile, destFile, width, height).get
+    fileRemoved
   }
 
   /**

src/main/scala/ch/memobase/FileUtils.scala

@@ -26,6 +26,7 @@ trait FileUtils {
   import models.Conversions._
 
   val rootPath: String
+  val cachedImagePath: String
 
   private val normalize: String => String =
     path => if (path.endsWith("/")) path.substring(0, path.length - 1) else path
@@ -42,7 +43,16 @@ trait FileUtils {
   val audioSnippetPath: String => String =
     id => s"${normalize(rootPath)}/$id-intro.mp3"
 
-  val imageFilePath: (String, MimeType) => String =
-    (id, mimeType) => s"${normalize(rootPath)}/$id.${getFileTypeExtension(mimeType).get}"
+  val imageFileRootPath: (String, MimeType) => String =
+    (id, mimeType) => imageFilePath(rootPath, id, mimeType, x => x)
+
+  val imageThumbnailFilePath: (String, MimeType) => String =
+    (id, mimeType) => imageFilePath(cachedImagePath, id, mimeType, x => x)
+
+  val posterImageThumbnailFilePath: (String, MimeType) => String =
+    (id, mimeType) => imageFilePath(cachedImagePath, id, mimeType, _.replace("/derived", "-poster"))
+
+  private val imageFilePath: (String, String, MimeType, String => String) => String =
+    (path, id, mimeType, idTransform) => s"${normalize(path)}/${idTransform(id)}.${getFileTypeExtension(mimeType).get}"
 
 }

src/main/scala/ch/memobase/MediaTransformations.scala

@@ -32,11 +32,15 @@ object MediaTransformations extends Logging {
   import sys.process._
 
   private def executeCommand(command: String): Try[Int] = Try {
+    logger.debug(s"Execute: $command")
     val stderr = StringBuilder.newBuilder
     val errorCode = command ! ProcessLogger(logger.debug(_), stderr append _)
     if (errorCode > 0) {
-      throw new IOException(s"application ${command.split(' ')(0)} exited with code $errorCode: ${stderr.toString()}")
+      val errorMsg = s"application ${command.split(' ')(0)} exited with code $errorCode: ${stderr.toString()}"
+      logger.warn(errorMsg)
+      throw new IOException(errorMsg)
     } else {
+      logger.debug("Command execution successful")
       errorCode
     }
   }
@@ -75,4 +79,38 @@ object MediaTransformations extends Logging {
       destFile
     }
   }
+
+  /**
+   * Creates an image thumbnail
+   *
+   * @param sourceFilePath Path to the source file
+   * @param destFile       Path to the final file
+   * @param width          Thumbnail's width. If None, width is set relative to height
+   * @param height         Thumbnail's height If None, height is set relative to width
+   * @return
+   */
+  def createImageThumbnail(sourceFilePath: String, destFile: String, width: Option[Int], height: Option[Int]): Try[String] = {
+    val externalCommand =
+      s"""convert
+         |-filter Triangle
+         |-define filter:support=2
+         |-thumbnail ${width.getOrElse("")}x${height.getOrElse("")}
+         |-auto-orient
+         |-unsharp 0.25x0.08+8.3+0.045
+         |-dither None
+         |-posterize 136
+         |-quality 82
+         |-define jpeg:fancy-upsampling=off
+         |-define png:compression-filter=5
+         |-define png:compression-level=9
+         |-define png:compression-strategy=1
+         |-define png:exclude-chunk=all
+         |-interlace none
+         |$sourceFilePath
+         |$destFile""".stripMargin.replaceAll("\n", " ")
+    Try {
+      executeCommand(externalCommand)
+      destFile
+    }
+  }
 }

src/main/scala/ch/memobase/RecordProcessor.scala

@@ -43,6 +43,7 @@ class RecordProcessor(fileHandler: DisseminationCopyHandler,
                       appSettings: Properties) extends FileUtils {
 
   val rootPath: String = appSettings.getProperty("mediaFolderRootPath")
+  val cachedImagePath: String = appSettings.getProperty("thumbnailFolderPath")
   val distributorUrl: String = appSettings.getProperty("distributorUrl")
   val maxRetries: Int = appSettings.getProperty("connectionMaxRetries").toInt
   val retryAfter: Int = appSettings.getProperty("connectionRetryAfterMs").toInt
@@ -92,13 +93,31 @@ class RecordProcessor(fileHandler: DisseminationCopyHandler,
       val res = fileHandler.createVideoCopy(data, destFile, mT)
       createOutcome(res, id, DigitalObject, destFile)
     case mT: ImageFile if resource == DigitalObject =>
-      val destFile = imageFilePath(id, mT)
-      val res = fileHandler.createImageCopy(data, destFile, mT)
-      createOutcome(res, id, DigitalObject, destFile)
-    case mT: ImageFile if resource == Thumbnail =>
+      val destFile = imageFileRootPath(id, mT)
+      createImageAndThumbnail(id, data, destFile, mT, DigitalObject, imageThumbnailFilePath)
+    case mT: ImageFile if resource == VideoThumbnail =>
       val destFile = videoPosterPath(id, mT)
-      val res = fileHandler.createImageCopy(data, destFile, mT)
-      createOutcome(res, id, Thumbnail, destFile)
+      createImageAndThumbnail(id, data, destFile, mT, VideoThumbnail, posterImageThumbnailFilePath)
+  }
+
+  private def createImageAndThumbnail(id: String,
+                                      data: ByteArrayOutputStream,
+                                      destImageFile: String,
+                                      mimeType: MimeType,
+                                      memobaseResource: MemobaseResource,
+                                      thumbnailFilePathFun: (String, MimeType) => String): List[ProcessOutcome] = {
+    val resMediaFile = fileHandler.createImageCopy(data, destImageFile, mimeType)
+    val outcomeMediaFile = createOutcome(resMediaFile, id, memobaseResource, destImageFile)
+    val destPreviewFile = thumbnailFilePathFun(id, mimeType)
+    val (width, height) = getThumbnailDimensions
+    val resThumbnail = fileHandler.createImageThumbnail(destImageFile, destPreviewFile, width, height)
+    val outcomeThumbnail = createOutcome(resThumbnail, id, ImageThumbnail, destPreviewFile)
+    outcomeMediaFile ++ outcomeThumbnail
+  }
+
+  private def getThumbnailDimensions: (Option[Int], Option[Int]) = {
+    (Try(appSettings.getProperty("thumbnailWidth").toInt).toOption.filter(_ >= 1),
+      Try(appSettings.getProperty("thumbnailHeight").toInt).toOption.filter(_ >= 1))
   }
 
   /**

src/main/scala/ch/memobase/RecordUtils.scala

@@ -38,7 +38,7 @@ trait RecordUtils {
 
   protected def isPreviewImage(obj: ujson.Obj): Boolean = {
     hasKeyValue(obj, "type") {
-      MemobaseResource(_) == Thumbnail
+      MemobaseResource(_) == VideoThumbnail
     }
   }