Unverified Commit fd1785c1 authored by Sebastian Schüpbach's avatar Sebastian Schüpbach
Browse files

improve report messages

parent a6c9ebd3
Pipeline #14628 passed with stages
in 12 minutes and 41 seconds
...@@ -57,7 +57,7 @@ object App extends scala.App with Logging { ...@@ -57,7 +57,7 @@ object App extends scala.App with Logging {
settings.getAppSettings.getProperty("fedoraUser"), settings.getAppSettings.getProperty("fedoraUser"),
settings.getAppSettings.getProperty("fedoraPassword") settings.getAppSettings.getProperty("fedoraPassword")
) )
val recordProcessor = new RecordProcessor(fileHandler, fCWrapper, settings.getAppSettings.getProperty("externalBaseUrl")) val recordProcessor = new RecordProcessor(fileHandler, fCWrapper, settings.getAppSettings)
val reporter = Reporter(settings.getKafkaProducerSettings, settings.getProcessReportTopic) val reporter = Reporter(settings.getKafkaProducerSettings, settings.getProcessReportTopic)
val consumerPollTimeoutMs = 100 val consumerPollTimeoutMs = 100
......
...@@ -71,20 +71,20 @@ class DisseminationCopyHandler(audioDestPath: String, imageDestPath: String, vid ...@@ -71,20 +71,20 @@ class DisseminationCopyHandler(audioDestPath: String, imageDestPath: String, vid
* Creates dissemination copy of audio file * Creates dissemination copy of audio file
* *
* @param data binary data as [[java.io.ByteArrayOutputStream]] instance * @param data binary data as [[java.io.ByteArrayOutputStream]] instance
* @param destId Filename of dissemination copy without extension * @param destFile Path to dissemination copy
* @param sourceFileType File type of input data * @param sourceFileType File type of input data
* @param isSnippet Process data as snippet * @param isSnippet Process data as snippet
* @return true if both files were overwritten, false otherwise * @return true if both files were overwritten, false otherwise
*/ */
def createAudioCopy(data: ByteArrayOutputStream, destId: String, sourceFileType: MimeType, isSnippet: Boolean = false): Try[Boolean] = Try { def createAudioCopy(data: ByteArrayOutputStream, destFile: String, sourceFileType: MimeType, isSnippet: Boolean = false): Try[Boolean] = Try {
// FIXME
val tempFilePath = Files.createTempFile("media-", "." + Conversions.getFileTypeExtension(sourceFileType).get) val tempFilePath = Files.createTempFile("media-", "." + Conversions.getFileTypeExtension(sourceFileType).get)
val destFile = Paths.get(audioDestPath, destId + (if (isSnippet) ".mp3" else ".mp4"))
writeData(data, tempFilePath) writeData(data, tempFilePath)
val copyRemoved = removeExistingFile(destFile) val copyRemoved = removeExistingFile(Paths.get(destFile))
if (isSnippet) { if (isSnippet) {
MediaTransformations.createAudioSnippet(tempFilePath.toString, destFile.toString, audioSnippetDuration) MediaTransformations.createAudioSnippet(tempFilePath.toString, destFile, audioSnippetDuration)
} else { } else {
MediaTransformations.audioToMp4(tempFilePath.toString, destFile.toString).get MediaTransformations.audioToMp4(tempFilePath.toString, destFile).get
} }
Files.delete(tempFilePath) Files.delete(tempFilePath)
copyRemoved copyRemoved
...@@ -94,16 +94,15 @@ class DisseminationCopyHandler(audioDestPath: String, imageDestPath: String, vid ...@@ -94,16 +94,15 @@ class DisseminationCopyHandler(audioDestPath: String, imageDestPath: String, vid
* Creates dissemination copy of image file * Creates dissemination copy of image file
* *
* @param data binary data as [[java.io.ByteArrayOutputStream]] instance * @param data binary data as [[java.io.ByteArrayOutputStream]] instance
* @param destId Filename of dissemination copy without extension * @param destFile Path to dissemination copy
* @param sourceFileType File type of input data * @param sourceFileType File type of input data
* @return true if copy was overwritten, false otherwise * @return true if copy was overwritten, false otherwise
*/ */
def createImageCopy(data: ByteArrayOutputStream, destId: String, sourceFileType: MimeType): Try[Boolean] = Try { def createImageCopy(data: ByteArrayOutputStream, destFile: String, sourceFileType: MimeType): Try[Boolean] = Try {
val tempFilePath = Files.createTempFile("media-", "." + Conversions.getFileTypeExtension(sourceFileType).get) val tempFilePath = Files.createTempFile("media-", "." + Conversions.getFileTypeExtension(sourceFileType).get)
val destFile = Paths.get(imageDestPath, destId + ".jp2")
writeData(data, tempFilePath) writeData(data, tempFilePath)
val copyRemoved = removeExistingFile(destFile) val copyRemoved = removeExistingFile(Paths.get(destFile))
MediaTransformations.imageToJp2(tempFilePath.toString, destFile.toString).get MediaTransformations.imageToJp2(tempFilePath.toString, destFile).get
Files.delete(tempFilePath) Files.delete(tempFilePath)
copyRemoved copyRemoved
} }
...@@ -112,50 +111,48 @@ class DisseminationCopyHandler(audioDestPath: String, imageDestPath: String, vid ...@@ -112,50 +111,48 @@ class DisseminationCopyHandler(audioDestPath: String, imageDestPath: String, vid
* Create dissemination copy of video file * Create dissemination copy of video file
* *
* @param data binary data as [[java.io.ByteArrayOutputStream]] instance * @param data binary data as [[java.io.ByteArrayOutputStream]] instance
* @param destId Filename of dissemination copy without extension * @param destFile Path to dissemination copy
* @param sourceFileType File type of input data * @param sourceFileType File type of input data
* @return true if copy was overwritten, false otherwise * @return true if copy was overwritten, false otherwise
*/ */
def createVideoCopy(data: ByteArrayOutputStream, destId: String, sourceFileType: MimeType): Try[Boolean] = Try { def createVideoCopy(data: ByteArrayOutputStream, destFile: String, sourceFileType: MimeType): Try[Boolean] = Try {
val destFile = Paths.get(videoDestPath, s"$destId.${Conversions.getFileTypeExtension(sourceFileType).get}") val destFileAsPath = Paths.get(destFile)
val copyRemoved = removeExistingFile(destFile) val copyRemoved = removeExistingFile(destFileAsPath)
writeData(data, destFile) writeData(data, destFileAsPath)
copyRemoved copyRemoved
} }
/** /**
* Deletes dissemination copy of audio file * Deletes dissemination copy of audio file
* *
* @param destId Filename of dissemination copy without extension * @param destFile Path to dissemination copy
* @return true if copy and snippet were deleted successfully, false otherwise * @return true if copy and snippet were deleted successfully, false otherwise
*/ */
def deleteAudioCopy(destId: String): Try[Boolean] = def deleteAudioCopy(destFile: String): Try[Boolean] =
Try { Try {
val res = Paths.get(audioDestPath, destId + ".mp4").toFile.delete() Paths.get(destFile).toFile.delete()
Paths.get(audioDestPath, destId + "-intro.mp3").toFile.delete() && res
} }
/** /**
* Deletes dissemination copy of image file * Deletes dissemination copy of image file
* *
* @param destId Filename of dissemination copy without extension * @param destFile Path to dissemination copy
* @return true if copy was deleted successfully, false otherwise * @return true if copy was deleted successfully, false otherwise
*/ */
def deleteImageCopy(destId: String): Try[Boolean] = def deleteImageCopy(destFile: String): Try[Boolean] =
Try { Try {
Paths.get(imageDestPath, destId + ".jp2").toFile.delete() Paths.get(destFile).toFile.delete()
} }
/** /**
* Deletes dissemination copy of video file * Deletes dissemination copy of video file
* *
* @param destId Filename of dissemination copy without extension * @param destFile Path to dissemination copy
* @param fileType File type of media
* @return true if copy was deleted successfully, false otherwise * @return true if copy was deleted successfully, false otherwise
*/ */
def deleteVideoCopy(destId: String, fileType: MimeType): Try[Boolean] = def deleteVideoCopy(destFile: String): Try[Boolean] =
Try { Try {
Paths.get(videoDestPath, s"$destId.${Conversions.getFileTypeExtension(fileType).get}").toFile.delete() Paths.get(destFile).toFile.delete()
} }
} }
/*
* Media Converter
* Extracts media files from Fedora repository
* Copyright (C) 2020 Memoriav
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package ch.memobase
import ch.memobase.models.MimeType
object FileUtils {
import models.Conversions._
val rootPath = "/media"
def createVideoFile(id: String, mimeType: MimeType): String =
s"$rootPath/$id.${getFileTypeExtension(mimeType).get}"
def createVideoPosterFile(id: String): String =
s"$rootPath/$id-poster.jp2"
def createAudioFile(id: String): String =
s"$rootPath/$id.mp4"
def createAudioSnippetFile(id: String): String =
s"$rootPath/$id-intro.mp3"
def createImageFile(id: String): String =
s"$rootPath/$id.jp2"
}
...@@ -21,11 +21,12 @@ package ch.memobase ...@@ -21,11 +21,12 @@ package ch.memobase
import java.io.ByteArrayOutputStream import java.io.ByteArrayOutputStream
import java.util.Properties
import ch.memobase.models._ import ch.memobase.models._
import org.apache.kafka.clients.consumer.ConsumerRecord import org.apache.kafka.clients.consumer.ConsumerRecord
import scala.util.{Failure, Success} import scala.util.{Failure, Success, Try}
trait ProcessOutcome trait ProcessOutcome
...@@ -36,10 +37,14 @@ case class ProcessFailure(id: String, msg: String, ex: Throwable) extends Proces ...@@ -36,10 +37,14 @@ case class ProcessFailure(id: String, msg: String, ex: Throwable) extends Proces
case class ProcessWarn(id: String, msg: String) extends ProcessOutcome case class ProcessWarn(id: String, msg: String) extends ProcessOutcome
class RecordProcessor(fileHandler: DisseminationCopyHandler, fedoraClientWrapper: FedoraClientWrapper, externalBaseUrl: String) { class RecordProcessor(fileHandler: DisseminationCopyHandler,
fedoraClientWrapper: FedoraClientWrapper,
appSettings: Properties) {
import FileUtils._
def process(record: ConsumerRecord[String, String]): List[ProcessOutcome] = { def process(record: ConsumerRecord[String, String]): List[ProcessOutcome] = {
BinaryResourceMetadata.build(record.value(), externalBaseUrl) flatMap { BinaryResourceMetadata.build(record.value(), appSettings.getProperty("externalBaseUrl")) flatMap {
case Success(binaryResource) => case Success(binaryResource) =>
handleBinaryResource(binaryResource, record.key()) handleBinaryResource(binaryResource, record.key())
case Failure(ex) => List(ex match { case Failure(ex) => List(ex match {
...@@ -65,48 +70,59 @@ class RecordProcessor(fileHandler: DisseminationCopyHandler, fedoraClientWrapper ...@@ -65,48 +70,59 @@ class RecordProcessor(fileHandler: DisseminationCopyHandler, fedoraClientWrapper
} }
} }
private def createOutcome(res: Try[Boolean], id: String, destFile: String): List[ProcessOutcome] = List(res match {
case Success(true) => ProcessSuccess(id, s"Updating of file $destFile successful")
case Success(false) => ProcessSuccess(id, s"Creation of file $destFile successful")
case Failure(ex) => ProcessFailure(id, s"Creation of file $destFile failed", ex)
})
private def deleteOutcome(res: Try[Boolean], id: String, destFile: String): List[ProcessOutcome] = List(res match {
case Success(true) => ProcessSuccess(id, s"Deletion of file $destFile successful")
case Success(false) => ProcessSuccess(id, s"No deletion of file $destFile because object does not exist")
case Failure(ex) => ProcessFailure(id, s"Deletion of file $destFile failed", ex)
})
private def deleteResource(id: String, private def deleteResource(id: String,
mimeType: MimeType, mimeType: MimeType,
instantiationType: Instantiation): List[ProcessOutcome] = { instantiationType: Instantiation): List[ProcessOutcome] = mimeType match {
(mimeType match { case _: AudioFile =>
case _: AudioFile => List(createAudioFile(id), createAudioSnippetFile(id))
(fileHandler.deleteAudioCopy(id), id) :: .map(path => (fileHandler.deleteAudioCopy(path), path))
List((fileHandler.deleteImageCopy(id + "-intro"), id)) .flatMap(x => deleteOutcome(x._1, id, x._2))
case mT: VideoFile => case mT: VideoFile =>
List((fileHandler.deleteVideoCopy(id, mT), id)) val destFile = createVideoFile(id, mT)
case _: ImageFile if instantiationType == DigitalObject => val res = fileHandler.deleteVideoCopy(destFile)
List((fileHandler.deleteImageCopy(id), id)) deleteOutcome(res, id, destFile)
case _: ImageFile if instantiationType == Thumbnail => case _: ImageFile if instantiationType == DigitalObject =>
val posterId = id + "-poster" val destFile = createImageFile(id)
List((fileHandler.deleteImageCopy(posterId), posterId)) val res = fileHandler.deleteImageCopy(destFile)
}) flatMap(res => List(res._1 match { deleteOutcome(res, id, destFile)
case Success(true) => ProcessSuccess(res._2, s"Deletion of object $id with type $mimeType successful") case _: ImageFile if instantiationType == Thumbnail =>
case Success(false) => ProcessSuccess(res._2, s"No deletion of object $id with type $mimeType because object does not exist") val destFile = createVideoPosterFile(id)
case Failure(ex) => ProcessFailure(res._2, "Deletion of object $id with type $mimeType failed", ex) val res = fileHandler.deleteImageCopy(destFile)
})) deleteOutcome(res, id, destFile)
} }
private def createResource(id: String, private def createResource(id: String,
mimeType: MimeType, mimeType: MimeType,
instantiationType: Instantiation, instantiationType: Instantiation,
data: ByteArrayOutputStream): List[ProcessOutcome] = { data: ByteArrayOutputStream): List[ProcessOutcome] = mimeType match {
(mimeType match {
case mT: AudioFile => case mT: AudioFile =>
(fileHandler.createAudioCopy(data, id, mT), id) :: List((createAudioFile(id), false), (createAudioSnippetFile(id), true))
List((fileHandler.createAudioCopy(data, id + "-intro", mT), id)) .map(path => (fileHandler.createAudioCopy(data, path._1, mT, path._2), path._1))
.flatMap(x => createOutcome(x._1, id, x._2))
case mT: VideoFile => case mT: VideoFile =>
List((fileHandler.createVideoCopy(data, id, mT), id)) val destFile = createVideoFile(id, mT)
val res = fileHandler.createVideoCopy(data, destFile, mT)
createOutcome(res, id, destFile)
case mT: ImageFile if instantiationType == DigitalObject => case mT: ImageFile if instantiationType == DigitalObject =>
List((fileHandler.createImageCopy(data, id, mT), id)) val destFile = createImageFile(id)
val res = fileHandler.createImageCopy(data, destFile, mT)
createOutcome(res, id, destFile)
case mT: ImageFile if instantiationType == Thumbnail => case mT: ImageFile if instantiationType == Thumbnail =>
val posterId = id + "-poster" val destFile = createVideoPosterFile(id)
List((fileHandler.createImageCopy(data, posterId, mT), posterId)) val res = fileHandler.createImageCopy(data, destFile, mT)
createOutcome(res, id, destFile)
}) flatMap(res => List(res._1 match {
case Success(true) => ProcessSuccess(res._2, s"Updating of object $id with type $mimeType successful")
case Success(false) => ProcessSuccess(res._2, s"Creation of object $id with type $mimeType successful")
case Failure(ex) => ProcessFailure(res._2, "Creation of object $id with type $mimeType failed", ex)
}))
} }
} }
...@@ -60,9 +60,10 @@ class DisseminationCopyHandlerTest extends AnyFunSuite with BeforeAndAfter { ...@@ -60,9 +60,10 @@ class DisseminationCopyHandlerTest extends AnyFunSuite with BeforeAndAfter {
data.write(buffer, 0, len) data.write(buffer, 0, len)
len = in.read(buffer) len = in.read(buffer)
} }
Paths.get(pathToTmpDir, destFileName).toFile.deleteOnExit() val destFile = Paths.get(pathToTmpDir, destFileName)
copyFun(data, destFileName.split('.')(0), fileType) destFile.toFile.deleteOnExit()
assert(Paths.get(pathToTmpDir, destFileName).toFile.exists()) copyFun(data, destFile.toString, fileType)
assert(destFile.toFile.exists())
} }
private def testAudioCopy(pathToTmpDir: String, private def testAudioCopy(pathToTmpDir: String,
...@@ -81,9 +82,10 @@ class DisseminationCopyHandlerTest extends AnyFunSuite with BeforeAndAfter { ...@@ -81,9 +82,10 @@ class DisseminationCopyHandlerTest extends AnyFunSuite with BeforeAndAfter {
data.write(buffer, 0, len) data.write(buffer, 0, len)
len = in.read(buffer) len = in.read(buffer)
} }
Paths.get(pathToTmpDir, destFileName).toFile.deleteOnExit() val destFile = Paths.get(pathToTmpDir, destFileName)
copyFun(data, destFileName.split('.')(0), fileType, isSnippet) destFile.toFile.deleteOnExit()
assert(Paths.get(pathToTmpDir, destFileName).toFile.exists()) copyFun(data, destFile.toString, fileType, isSnippet)
assert(destFile.toFile.exists())
} }
...@@ -91,7 +93,8 @@ class DisseminationCopyHandlerTest extends AnyFunSuite with BeforeAndAfter { ...@@ -91,7 +93,8 @@ class DisseminationCopyHandlerTest extends AnyFunSuite with BeforeAndAfter {
val f = fixture val f = fixture
val testFile = Files.createFile(Paths.get(f.resPath, "test.mp4")) val testFile = Files.createFile(Paths.get(f.resPath, "test.mp4"))
val testSnippetFile = Files.createFile(Paths.get(f.resPath, "test-intro.mp3")) val testSnippetFile = Files.createFile(Paths.get(f.resPath, "test-intro.mp3"))
f.fileHandler.deleteAudioCopy("test") f.fileHandler.deleteAudioCopy(testFile.toString)
f.fileHandler.deleteAudioCopy(testSnippetFile.toString)
assert(!testFile.toFile.exists()) assert(!testFile.toFile.exists())
assert(!testSnippetFile.toFile.exists()) assert(!testSnippetFile.toFile.exists())
} }
......
...@@ -20,6 +20,7 @@ ...@@ -20,6 +20,7 @@
package ch.memobase package ch.memobase
import java.io.ByteArrayOutputStream import java.io.ByteArrayOutputStream
import java.util.Properties
import ch.memobase.models._ import ch.memobase.models._
import org.apache.kafka.clients.consumer.ConsumerRecord import org.apache.kafka.clients.consumer.ConsumerRecord
...@@ -110,7 +111,9 @@ class RecordProcessorTest extends AnyFunSuite with MockFactory { ...@@ -110,7 +111,9 @@ class RecordProcessorTest extends AnyFunSuite with MockFactory {
val mockFCW = f.mockFedoraClientWrapper val mockFCW = f.mockFedoraClientWrapper
val (incomingMessage, _) = createIncomingMessage(Delete, Mp3File, hasBinaryResource = false) val (incomingMessage, _) = createIncomingMessage(Delete, Mp3File, hasBinaryResource = false)
(mockFCW.fetchBinaryResource _).expects(*).never() (mockFCW.fetchBinaryResource _).expects(*).never()
val rP = new RecordProcessor(mockDCH, mockFCW, f.externalBaseUrl) val props = new Properties()
props.setProperty("externalBaseUrl", f.externalBaseUrl)
val rP = new RecordProcessor(mockDCH, mockFCW, props)
val cR = new ConsumerRecord[String, String]("_void", 1, 0, "1", incomingMessage) val cR = new ConsumerRecord[String, String]("_void", 1, 0, "1", incomingMessage)
rP.process(cR) rP.process(cR)
} }
......
File added
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment