Unverified Commit fd1785c1 authored by Sebastian Schüpbach's avatar Sebastian Schüpbach
Browse files

improve report messages

parent a6c9ebd3
Pipeline #14628 passed with stages
in 12 minutes and 41 seconds
......@@ -57,7 +57,7 @@ object App extends scala.App with Logging {
settings.getAppSettings.getProperty("fedoraUser"),
settings.getAppSettings.getProperty("fedoraPassword")
)
val recordProcessor = new RecordProcessor(fileHandler, fCWrapper, settings.getAppSettings.getProperty("externalBaseUrl"))
val recordProcessor = new RecordProcessor(fileHandler, fCWrapper, settings.getAppSettings)
val reporter = Reporter(settings.getKafkaProducerSettings, settings.getProcessReportTopic)
val consumerPollTimeoutMs = 100
......
......@@ -71,20 +71,20 @@ class DisseminationCopyHandler(audioDestPath: String, imageDestPath: String, vid
* Creates dissemination copy of audio file
*
* @param data binary data as [[java.io.ByteArrayOutputStream]] instance
* @param destId Filename of dissemination copy without extension
* @param destFile Path to dissemination copy
* @param sourceFileType File type of input data
* @param isSnippet Process data as snippet
* @return true if both files were overwritten, false otherwise
*/
def createAudioCopy(data: ByteArrayOutputStream, destId: String, sourceFileType: MimeType, isSnippet: Boolean = false): Try[Boolean] = Try {
def createAudioCopy(data: ByteArrayOutputStream, destFile: String, sourceFileType: MimeType, isSnippet: Boolean = false): Try[Boolean] = Try {
// FIXME
val tempFilePath = Files.createTempFile("media-", "." + Conversions.getFileTypeExtension(sourceFileType).get)
val destFile = Paths.get(audioDestPath, destId + (if (isSnippet) ".mp3" else ".mp4"))
writeData(data, tempFilePath)
val copyRemoved = removeExistingFile(destFile)
val copyRemoved = removeExistingFile(Paths.get(destFile))
if (isSnippet) {
MediaTransformations.createAudioSnippet(tempFilePath.toString, destFile.toString, audioSnippetDuration)
MediaTransformations.createAudioSnippet(tempFilePath.toString, destFile, audioSnippetDuration)
} else {
MediaTransformations.audioToMp4(tempFilePath.toString, destFile.toString).get
MediaTransformations.audioToMp4(tempFilePath.toString, destFile).get
}
Files.delete(tempFilePath)
copyRemoved
......@@ -94,16 +94,15 @@ class DisseminationCopyHandler(audioDestPath: String, imageDestPath: String, vid
* Creates dissemination copy of image file
*
* @param data binary data as [[java.io.ByteArrayOutputStream]] instance
* @param destId Filename of dissemination copy without extension
* @param destFile Path to dissemination copy
* @param sourceFileType File type of input data
* @return true if copy was overwritten, false otherwise
*/
def createImageCopy(data: ByteArrayOutputStream, destId: String, sourceFileType: MimeType): Try[Boolean] = Try {
def createImageCopy(data: ByteArrayOutputStream, destFile: String, sourceFileType: MimeType): Try[Boolean] = Try {
val tempFilePath = Files.createTempFile("media-", "." + Conversions.getFileTypeExtension(sourceFileType).get)
val destFile = Paths.get(imageDestPath, destId + ".jp2")
writeData(data, tempFilePath)
val copyRemoved = removeExistingFile(destFile)
MediaTransformations.imageToJp2(tempFilePath.toString, destFile.toString).get
val copyRemoved = removeExistingFile(Paths.get(destFile))
MediaTransformations.imageToJp2(tempFilePath.toString, destFile).get
Files.delete(tempFilePath)
copyRemoved
}
......@@ -112,50 +111,48 @@ class DisseminationCopyHandler(audioDestPath: String, imageDestPath: String, vid
* Create dissemination copy of video file
*
* @param data binary data as [[java.io.ByteArrayOutputStream]] instance
* @param destId Filename of dissemination copy without extension
* @param destFile Path to dissemination copy
* @param sourceFileType File type of input data
* @return true if copy was overwritten, false otherwise
*/
def createVideoCopy(data: ByteArrayOutputStream, destId: String, sourceFileType: MimeType): Try[Boolean] = Try {
val destFile = Paths.get(videoDestPath, s"$destId.${Conversions.getFileTypeExtension(sourceFileType).get}")
val copyRemoved = removeExistingFile(destFile)
writeData(data, destFile)
def createVideoCopy(data: ByteArrayOutputStream, destFile: String, sourceFileType: MimeType): Try[Boolean] = Try {
val destFileAsPath = Paths.get(destFile)
val copyRemoved = removeExistingFile(destFileAsPath)
writeData(data, destFileAsPath)
copyRemoved
}
/**
* Deletes dissemination copy of audio file
*
* @param destId Filename of dissemination copy without extension
* @param destFile Path to dissemination copy
* @return true if copy and snippet were deleted successfully, false otherwise
*/
def deleteAudioCopy(destId: String): Try[Boolean] =
def deleteAudioCopy(destFile: String): Try[Boolean] =
Try {
val res = Paths.get(audioDestPath, destId + ".mp4").toFile.delete()
Paths.get(audioDestPath, destId + "-intro.mp3").toFile.delete() && res
Paths.get(destFile).toFile.delete()
}
/**
* Deletes dissemination copy of image file
*
* @param destId Filename of dissemination copy without extension
* @param destFile Path to dissemination copy
* @return true if copy was deleted successfully, false otherwise
*/
def deleteImageCopy(destId: String): Try[Boolean] =
def deleteImageCopy(destFile: String): Try[Boolean] =
Try {
Paths.get(imageDestPath, destId + ".jp2").toFile.delete()
Paths.get(destFile).toFile.delete()
}
/**
* Deletes dissemination copy of video file
*
* @param destId Filename of dissemination copy without extension
* @param fileType File type of media
* @param destFile Path to dissemination copy
* @return true if copy was deleted successfully, false otherwise
*/
def deleteVideoCopy(destId: String, fileType: MimeType): Try[Boolean] =
def deleteVideoCopy(destFile: String): Try[Boolean] =
Try {
Paths.get(videoDestPath, s"$destId.${Conversions.getFileTypeExtension(fileType).get}").toFile.delete()
Paths.get(destFile).toFile.delete()
}
}
/*
* Media Converter
* Extracts media files from Fedora repository
* Copyright (C) 2020 Memoriav
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package ch.memobase
import ch.memobase.models.MimeType
object FileUtils {
import models.Conversions._
val rootPath = "/media"
def createVideoFile(id: String, mimeType: MimeType): String =
s"$rootPath/$id.${getFileTypeExtension(mimeType).get}"
def createVideoPosterFile(id: String): String =
s"$rootPath/$id-poster.jp2"
def createAudioFile(id: String): String =
s"$rootPath/$id.mp4"
def createAudioSnippetFile(id: String): String =
s"$rootPath/$id-intro.mp3"
def createImageFile(id: String): String =
s"$rootPath/$id.jp2"
}
......@@ -21,11 +21,12 @@ package ch.memobase
import java.io.ByteArrayOutputStream
import java.util.Properties
import ch.memobase.models._
import org.apache.kafka.clients.consumer.ConsumerRecord
import scala.util.{Failure, Success}
import scala.util.{Failure, Success, Try}
trait ProcessOutcome
......@@ -36,10 +37,14 @@ case class ProcessFailure(id: String, msg: String, ex: Throwable) extends Proces
case class ProcessWarn(id: String, msg: String) extends ProcessOutcome
class RecordProcessor(fileHandler: DisseminationCopyHandler, fedoraClientWrapper: FedoraClientWrapper, externalBaseUrl: String) {
class RecordProcessor(fileHandler: DisseminationCopyHandler,
fedoraClientWrapper: FedoraClientWrapper,
appSettings: Properties) {
import FileUtils._
def process(record: ConsumerRecord[String, String]): List[ProcessOutcome] = {
BinaryResourceMetadata.build(record.value(), externalBaseUrl) flatMap {
BinaryResourceMetadata.build(record.value(), appSettings.getProperty("externalBaseUrl")) flatMap {
case Success(binaryResource) =>
handleBinaryResource(binaryResource, record.key())
case Failure(ex) => List(ex match {
......@@ -65,48 +70,59 @@ class RecordProcessor(fileHandler: DisseminationCopyHandler, fedoraClientWrapper
}
}
private def createOutcome(res: Try[Boolean], id: String, destFile: String): List[ProcessOutcome] = List(res match {
case Success(true) => ProcessSuccess(id, s"Updating of file $destFile successful")
case Success(false) => ProcessSuccess(id, s"Creation of file $destFile successful")
case Failure(ex) => ProcessFailure(id, s"Creation of file $destFile failed", ex)
})
private def deleteOutcome(res: Try[Boolean], id: String, destFile: String): List[ProcessOutcome] = List(res match {
case Success(true) => ProcessSuccess(id, s"Deletion of file $destFile successful")
case Success(false) => ProcessSuccess(id, s"No deletion of file $destFile because object does not exist")
case Failure(ex) => ProcessFailure(id, s"Deletion of file $destFile failed", ex)
})
private def deleteResource(id: String,
mimeType: MimeType,
instantiationType: Instantiation): List[ProcessOutcome] = {
(mimeType match {
instantiationType: Instantiation): List[ProcessOutcome] = mimeType match {
case _: AudioFile =>
(fileHandler.deleteAudioCopy(id), id) ::
List((fileHandler.deleteImageCopy(id + "-intro"), id))
List(createAudioFile(id), createAudioSnippetFile(id))
.map(path => (fileHandler.deleteAudioCopy(path), path))
.flatMap(x => deleteOutcome(x._1, id, x._2))
case mT: VideoFile =>
List((fileHandler.deleteVideoCopy(id, mT), id))
val destFile = createVideoFile(id, mT)
val res = fileHandler.deleteVideoCopy(destFile)
deleteOutcome(res, id, destFile)
case _: ImageFile if instantiationType == DigitalObject =>
List((fileHandler.deleteImageCopy(id), id))
val destFile = createImageFile(id)
val res = fileHandler.deleteImageCopy(destFile)
deleteOutcome(res, id, destFile)
case _: ImageFile if instantiationType == Thumbnail =>
val posterId = id + "-poster"
List((fileHandler.deleteImageCopy(posterId), posterId))
}) flatMap(res => List(res._1 match {
case Success(true) => ProcessSuccess(res._2, s"Deletion of object $id with type $mimeType successful")
case Success(false) => ProcessSuccess(res._2, s"No deletion of object $id with type $mimeType because object does not exist")
case Failure(ex) => ProcessFailure(res._2, "Deletion of object $id with type $mimeType failed", ex)
}))
val destFile = createVideoPosterFile(id)
val res = fileHandler.deleteImageCopy(destFile)
deleteOutcome(res, id, destFile)
}
private def createResource(id: String,
mimeType: MimeType,
instantiationType: Instantiation,
data: ByteArrayOutputStream): List[ProcessOutcome] = {
(mimeType match {
data: ByteArrayOutputStream): List[ProcessOutcome] = mimeType match {
case mT: AudioFile =>
(fileHandler.createAudioCopy(data, id, mT), id) ::
List((fileHandler.createAudioCopy(data, id + "-intro", mT), id))
List((createAudioFile(id), false), (createAudioSnippetFile(id), true))
.map(path => (fileHandler.createAudioCopy(data, path._1, mT, path._2), path._1))
.flatMap(x => createOutcome(x._1, id, x._2))
case mT: VideoFile =>
List((fileHandler.createVideoCopy(data, id, mT), id))
val destFile = createVideoFile(id, mT)
val res = fileHandler.createVideoCopy(data, destFile, mT)
createOutcome(res, id, destFile)
case mT: ImageFile if instantiationType == DigitalObject =>
List((fileHandler.createImageCopy(data, id, mT), id))
val destFile = createImageFile(id)
val res = fileHandler.createImageCopy(data, destFile, mT)
createOutcome(res, id, destFile)
case mT: ImageFile if instantiationType == Thumbnail =>
val posterId = id + "-poster"
List((fileHandler.createImageCopy(data, posterId, mT), posterId))
}) flatMap(res => List(res._1 match {
case Success(true) => ProcessSuccess(res._2, s"Updating of object $id with type $mimeType successful")
case Success(false) => ProcessSuccess(res._2, s"Creation of object $id with type $mimeType successful")
case Failure(ex) => ProcessFailure(res._2, "Creation of object $id with type $mimeType failed", ex)
}))
val destFile = createVideoPosterFile(id)
val res = fileHandler.createImageCopy(data, destFile, mT)
createOutcome(res, id, destFile)
}
}
......@@ -60,9 +60,10 @@ class DisseminationCopyHandlerTest extends AnyFunSuite with BeforeAndAfter {
data.write(buffer, 0, len)
len = in.read(buffer)
}
Paths.get(pathToTmpDir, destFileName).toFile.deleteOnExit()
copyFun(data, destFileName.split('.')(0), fileType)
assert(Paths.get(pathToTmpDir, destFileName).toFile.exists())
val destFile = Paths.get(pathToTmpDir, destFileName)
destFile.toFile.deleteOnExit()
copyFun(data, destFile.toString, fileType)
assert(destFile.toFile.exists())
}
private def testAudioCopy(pathToTmpDir: String,
......@@ -81,9 +82,10 @@ class DisseminationCopyHandlerTest extends AnyFunSuite with BeforeAndAfter {
data.write(buffer, 0, len)
len = in.read(buffer)
}
Paths.get(pathToTmpDir, destFileName).toFile.deleteOnExit()
copyFun(data, destFileName.split('.')(0), fileType, isSnippet)
assert(Paths.get(pathToTmpDir, destFileName).toFile.exists())
val destFile = Paths.get(pathToTmpDir, destFileName)
destFile.toFile.deleteOnExit()
copyFun(data, destFile.toString, fileType, isSnippet)
assert(destFile.toFile.exists())
}
......@@ -91,7 +93,8 @@ class DisseminationCopyHandlerTest extends AnyFunSuite with BeforeAndAfter {
val f = fixture
val testFile = Files.createFile(Paths.get(f.resPath, "test.mp4"))
val testSnippetFile = Files.createFile(Paths.get(f.resPath, "test-intro.mp3"))
f.fileHandler.deleteAudioCopy("test")
f.fileHandler.deleteAudioCopy(testFile.toString)
f.fileHandler.deleteAudioCopy(testSnippetFile.toString)
assert(!testFile.toFile.exists())
assert(!testSnippetFile.toFile.exists())
}
......
......@@ -20,6 +20,7 @@
package ch.memobase
import java.io.ByteArrayOutputStream
import java.util.Properties
import ch.memobase.models._
import org.apache.kafka.clients.consumer.ConsumerRecord
......@@ -110,7 +111,9 @@ class RecordProcessorTest extends AnyFunSuite with MockFactory {
val mockFCW = f.mockFedoraClientWrapper
val (incomingMessage, _) = createIncomingMessage(Delete, Mp3File, hasBinaryResource = false)
(mockFCW.fetchBinaryResource _).expects(*).never()
val rP = new RecordProcessor(mockDCH, mockFCW, f.externalBaseUrl)
val props = new Properties()
props.setProperty("externalBaseUrl", f.externalBaseUrl)
val rP = new RecordProcessor(mockDCH, mockFCW, props)
val cR = new ConsumerRecord[String, String]("_void", 1, 0, "1", incomingMessage)
rP.process(cR)
}
......
File added
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment