In order to mitigate against the brute force attacks against Gitlab accounts, we are moving to all edu-ID Logins. We would like to remind you to link your account with your edu-id. Login will be possible only by edu-ID after November 30, 2021. Here you can find the instructions for linking your account.

If you don't have a SWITCH edu-ID, you can create one with this guide here

kind regards

This Server has been upgraded to GitLab release 14.2.6

Unverified Commit fd1785c1 authored by Sebastian Schüpbach's avatar Sebastian Schüpbach
Browse files

improve report messages

parent a6c9ebd3
Pipeline #14628 passed with stages
in 12 minutes and 41 seconds
......@@ -57,7 +57,7 @@ object App extends scala.App with Logging {
settings.getAppSettings.getProperty("fedoraUser"),
settings.getAppSettings.getProperty("fedoraPassword")
)
val recordProcessor = new RecordProcessor(fileHandler, fCWrapper, settings.getAppSettings.getProperty("externalBaseUrl"))
val recordProcessor = new RecordProcessor(fileHandler, fCWrapper, settings.getAppSettings)
val reporter = Reporter(settings.getKafkaProducerSettings, settings.getProcessReportTopic)
val consumerPollTimeoutMs = 100
......
......@@ -71,20 +71,20 @@ class DisseminationCopyHandler(audioDestPath: String, imageDestPath: String, vid
* Creates dissemination copy of audio file
*
* @param data binary data as [[java.io.ByteArrayOutputStream]] instance
* @param destId Filename of dissemination copy without extension
* @param destFile Path to dissemination copy
* @param sourceFileType File type of input data
* @param isSnippet Process data as snippet
* @return true if both files were overwritten, false otherwise
*/
def createAudioCopy(data: ByteArrayOutputStream, destId: String, sourceFileType: MimeType, isSnippet: Boolean = false): Try[Boolean] = Try {
def createAudioCopy(data: ByteArrayOutputStream, destFile: String, sourceFileType: MimeType, isSnippet: Boolean = false): Try[Boolean] = Try {
// FIXME
val tempFilePath = Files.createTempFile("media-", "." + Conversions.getFileTypeExtension(sourceFileType).get)
val destFile = Paths.get(audioDestPath, destId + (if (isSnippet) ".mp3" else ".mp4"))
writeData(data, tempFilePath)
val copyRemoved = removeExistingFile(destFile)
val copyRemoved = removeExistingFile(Paths.get(destFile))
if (isSnippet) {
MediaTransformations.createAudioSnippet(tempFilePath.toString, destFile.toString, audioSnippetDuration)
MediaTransformations.createAudioSnippet(tempFilePath.toString, destFile, audioSnippetDuration)
} else {
MediaTransformations.audioToMp4(tempFilePath.toString, destFile.toString).get
MediaTransformations.audioToMp4(tempFilePath.toString, destFile).get
}
Files.delete(tempFilePath)
copyRemoved
......@@ -94,16 +94,15 @@ class DisseminationCopyHandler(audioDestPath: String, imageDestPath: String, vid
* Creates dissemination copy of image file
*
* @param data binary data as [[java.io.ByteArrayOutputStream]] instance
* @param destId Filename of dissemination copy without extension
* @param destFile Path to dissemination copy
* @param sourceFileType File type of input data
* @return true if copy was overwritten, false otherwise
*/
def createImageCopy(data: ByteArrayOutputStream, destId: String, sourceFileType: MimeType): Try[Boolean] = Try {
def createImageCopy(data: ByteArrayOutputStream, destFile: String, sourceFileType: MimeType): Try[Boolean] = Try {
val tempFilePath = Files.createTempFile("media-", "." + Conversions.getFileTypeExtension(sourceFileType).get)
val destFile = Paths.get(imageDestPath, destId + ".jp2")
writeData(data, tempFilePath)
val copyRemoved = removeExistingFile(destFile)
MediaTransformations.imageToJp2(tempFilePath.toString, destFile.toString).get
val copyRemoved = removeExistingFile(Paths.get(destFile))
MediaTransformations.imageToJp2(tempFilePath.toString, destFile).get
Files.delete(tempFilePath)
copyRemoved
}
......@@ -112,50 +111,48 @@ class DisseminationCopyHandler(audioDestPath: String, imageDestPath: String, vid
* Create dissemination copy of video file
*
* @param data binary data as [[java.io.ByteArrayOutputStream]] instance
* @param destId Filename of dissemination copy without extension
* @param destFile Path to dissemination copy
* @param sourceFileType File type of input data
* @return true if copy was overwritten, false otherwise
*/
def createVideoCopy(data: ByteArrayOutputStream, destId: String, sourceFileType: MimeType): Try[Boolean] = Try {
val destFile = Paths.get(videoDestPath, s"$destId.${Conversions.getFileTypeExtension(sourceFileType).get}")
val copyRemoved = removeExistingFile(destFile)
writeData(data, destFile)
def createVideoCopy(data: ByteArrayOutputStream, destFile: String, sourceFileType: MimeType): Try[Boolean] = Try {
val destFileAsPath = Paths.get(destFile)
val copyRemoved = removeExistingFile(destFileAsPath)
writeData(data, destFileAsPath)
copyRemoved
}
/**
* Deletes dissemination copy of audio file
*
* @param destId Filename of dissemination copy without extension
* @param destFile Path to dissemination copy
* @return true if copy and snippet were deleted successfully, false otherwise
*/
def deleteAudioCopy(destId: String): Try[Boolean] =
def deleteAudioCopy(destFile: String): Try[Boolean] =
Try {
val res = Paths.get(audioDestPath, destId + ".mp4").toFile.delete()
Paths.get(audioDestPath, destId + "-intro.mp3").toFile.delete() && res
Paths.get(destFile).toFile.delete()
}
/**
* Deletes dissemination copy of image file
*
* @param destId Filename of dissemination copy without extension
* @param destFile Path to dissemination copy
* @return true if copy was deleted successfully, false otherwise
*/
def deleteImageCopy(destId: String): Try[Boolean] =
def deleteImageCopy(destFile: String): Try[Boolean] =
Try {
Paths.get(imageDestPath, destId + ".jp2").toFile.delete()
Paths.get(destFile).toFile.delete()
}
/**
* Deletes dissemination copy of video file
*
* @param destId Filename of dissemination copy without extension
* @param fileType File type of media
* @param destFile Path to dissemination copy
* @return true if copy was deleted successfully, false otherwise
*/
def deleteVideoCopy(destId: String, fileType: MimeType): Try[Boolean] =
def deleteVideoCopy(destFile: String): Try[Boolean] =
Try {
Paths.get(videoDestPath, s"$destId.${Conversions.getFileTypeExtension(fileType).get}").toFile.delete()
Paths.get(destFile).toFile.delete()
}
}
/*
* Media Converter
* Extracts media files from Fedora repository
* Copyright (C) 2020 Memoriav
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package ch.memobase
import ch.memobase.models.MimeType
object FileUtils {
import models.Conversions._
val rootPath = "/media"
def createVideoFile(id: String, mimeType: MimeType): String =
s"$rootPath/$id.${getFileTypeExtension(mimeType).get}"
def createVideoPosterFile(id: String): String =
s"$rootPath/$id-poster.jp2"
def createAudioFile(id: String): String =
s"$rootPath/$id.mp4"
def createAudioSnippetFile(id: String): String =
s"$rootPath/$id-intro.mp3"
def createImageFile(id: String): String =
s"$rootPath/$id.jp2"
}
......@@ -21,11 +21,12 @@ package ch.memobase
import java.io.ByteArrayOutputStream
import java.util.Properties
import ch.memobase.models._
import org.apache.kafka.clients.consumer.ConsumerRecord
import scala.util.{Failure, Success}
import scala.util.{Failure, Success, Try}
trait ProcessOutcome
......@@ -36,10 +37,14 @@ case class ProcessFailure(id: String, msg: String, ex: Throwable) extends Proces
case class ProcessWarn(id: String, msg: String) extends ProcessOutcome
class RecordProcessor(fileHandler: DisseminationCopyHandler, fedoraClientWrapper: FedoraClientWrapper, externalBaseUrl: String) {
class RecordProcessor(fileHandler: DisseminationCopyHandler,
fedoraClientWrapper: FedoraClientWrapper,
appSettings: Properties) {
import FileUtils._
def process(record: ConsumerRecord[String, String]): List[ProcessOutcome] = {
BinaryResourceMetadata.build(record.value(), externalBaseUrl) flatMap {
BinaryResourceMetadata.build(record.value(), appSettings.getProperty("externalBaseUrl")) flatMap {
case Success(binaryResource) =>
handleBinaryResource(binaryResource, record.key())
case Failure(ex) => List(ex match {
......@@ -65,48 +70,59 @@ class RecordProcessor(fileHandler: DisseminationCopyHandler, fedoraClientWrapper
}
}
private def createOutcome(res: Try[Boolean], id: String, destFile: String): List[ProcessOutcome] = List(res match {
case Success(true) => ProcessSuccess(id, s"Updating of file $destFile successful")
case Success(false) => ProcessSuccess(id, s"Creation of file $destFile successful")
case Failure(ex) => ProcessFailure(id, s"Creation of file $destFile failed", ex)
})
private def deleteOutcome(res: Try[Boolean], id: String, destFile: String): List[ProcessOutcome] = List(res match {
case Success(true) => ProcessSuccess(id, s"Deletion of file $destFile successful")
case Success(false) => ProcessSuccess(id, s"No deletion of file $destFile because object does not exist")
case Failure(ex) => ProcessFailure(id, s"Deletion of file $destFile failed", ex)
})
private def deleteResource(id: String,
mimeType: MimeType,
instantiationType: Instantiation): List[ProcessOutcome] = {
(mimeType match {
case _: AudioFile =>
(fileHandler.deleteAudioCopy(id), id) ::
List((fileHandler.deleteImageCopy(id + "-intro"), id))
case mT: VideoFile =>
List((fileHandler.deleteVideoCopy(id, mT), id))
case _: ImageFile if instantiationType == DigitalObject =>
List((fileHandler.deleteImageCopy(id), id))
case _: ImageFile if instantiationType == Thumbnail =>
val posterId = id + "-poster"
List((fileHandler.deleteImageCopy(posterId), posterId))
}) flatMap(res => List(res._1 match {
case Success(true) => ProcessSuccess(res._2, s"Deletion of object $id with type $mimeType successful")
case Success(false) => ProcessSuccess(res._2, s"No deletion of object $id with type $mimeType because object does not exist")
case Failure(ex) => ProcessFailure(res._2, "Deletion of object $id with type $mimeType failed", ex)
}))
instantiationType: Instantiation): List[ProcessOutcome] = mimeType match {
case _: AudioFile =>
List(createAudioFile(id), createAudioSnippetFile(id))
.map(path => (fileHandler.deleteAudioCopy(path), path))
.flatMap(x => deleteOutcome(x._1, id, x._2))
case mT: VideoFile =>
val destFile = createVideoFile(id, mT)
val res = fileHandler.deleteVideoCopy(destFile)
deleteOutcome(res, id, destFile)
case _: ImageFile if instantiationType == DigitalObject =>
val destFile = createImageFile(id)
val res = fileHandler.deleteImageCopy(destFile)
deleteOutcome(res, id, destFile)
case _: ImageFile if instantiationType == Thumbnail =>
val destFile = createVideoPosterFile(id)
val res = fileHandler.deleteImageCopy(destFile)
deleteOutcome(res, id, destFile)
}
private def createResource(id: String,
mimeType: MimeType,
instantiationType: Instantiation,
data: ByteArrayOutputStream): List[ProcessOutcome] = {
(mimeType match {
data: ByteArrayOutputStream): List[ProcessOutcome] = mimeType match {
case mT: AudioFile =>
(fileHandler.createAudioCopy(data, id, mT), id) ::
List((fileHandler.createAudioCopy(data, id + "-intro", mT), id))
List((createAudioFile(id), false), (createAudioSnippetFile(id), true))
.map(path => (fileHandler.createAudioCopy(data, path._1, mT, path._2), path._1))
.flatMap(x => createOutcome(x._1, id, x._2))
case mT: VideoFile =>
List((fileHandler.createVideoCopy(data, id, mT), id))
val destFile = createVideoFile(id, mT)
val res = fileHandler.createVideoCopy(data, destFile, mT)
createOutcome(res, id, destFile)
case mT: ImageFile if instantiationType == DigitalObject =>
List((fileHandler.createImageCopy(data, id, mT), id))
val destFile = createImageFile(id)
val res = fileHandler.createImageCopy(data, destFile, mT)
createOutcome(res, id, destFile)
case mT: ImageFile if instantiationType == Thumbnail =>
val posterId = id + "-poster"
List((fileHandler.createImageCopy(data, posterId, mT), posterId))
}) flatMap(res => List(res._1 match {
case Success(true) => ProcessSuccess(res._2, s"Updating of object $id with type $mimeType successful")
case Success(false) => ProcessSuccess(res._2, s"Creation of object $id with type $mimeType successful")
case Failure(ex) => ProcessFailure(res._2, "Creation of object $id with type $mimeType failed", ex)
}))
val destFile = createVideoPosterFile(id)
val res = fileHandler.createImageCopy(data, destFile, mT)
createOutcome(res, id, destFile)
}
}
......@@ -60,9 +60,10 @@ class DisseminationCopyHandlerTest extends AnyFunSuite with BeforeAndAfter {
data.write(buffer, 0, len)
len = in.read(buffer)
}
Paths.get(pathToTmpDir, destFileName).toFile.deleteOnExit()
copyFun(data, destFileName.split('.')(0), fileType)
assert(Paths.get(pathToTmpDir, destFileName).toFile.exists())
val destFile = Paths.get(pathToTmpDir, destFileName)
destFile.toFile.deleteOnExit()
copyFun(data, destFile.toString, fileType)
assert(destFile.toFile.exists())
}
private def testAudioCopy(pathToTmpDir: String,
......@@ -81,9 +82,10 @@ class DisseminationCopyHandlerTest extends AnyFunSuite with BeforeAndAfter {
data.write(buffer, 0, len)
len = in.read(buffer)
}
Paths.get(pathToTmpDir, destFileName).toFile.deleteOnExit()
copyFun(data, destFileName.split('.')(0), fileType, isSnippet)
assert(Paths.get(pathToTmpDir, destFileName).toFile.exists())
val destFile = Paths.get(pathToTmpDir, destFileName)
destFile.toFile.deleteOnExit()
copyFun(data, destFile.toString, fileType, isSnippet)
assert(destFile.toFile.exists())
}
......@@ -91,7 +93,8 @@ class DisseminationCopyHandlerTest extends AnyFunSuite with BeforeAndAfter {
val f = fixture
val testFile = Files.createFile(Paths.get(f.resPath, "test.mp4"))
val testSnippetFile = Files.createFile(Paths.get(f.resPath, "test-intro.mp3"))
f.fileHandler.deleteAudioCopy("test")
f.fileHandler.deleteAudioCopy(testFile.toString)
f.fileHandler.deleteAudioCopy(testSnippetFile.toString)
assert(!testFile.toFile.exists())
assert(!testSnippetFile.toFile.exists())
}
......
......@@ -20,6 +20,7 @@
package ch.memobase
import java.io.ByteArrayOutputStream
import java.util.Properties
import ch.memobase.models._
import org.apache.kafka.clients.consumer.ConsumerRecord
......@@ -110,7 +111,9 @@ class RecordProcessorTest extends AnyFunSuite with MockFactory {
val mockFCW = f.mockFedoraClientWrapper
val (incomingMessage, _) = createIncomingMessage(Delete, Mp3File, hasBinaryResource = false)
(mockFCW.fetchBinaryResource _).expects(*).never()
val rP = new RecordProcessor(mockDCH, mockFCW, f.externalBaseUrl)
val props = new Properties()
props.setProperty("externalBaseUrl", f.externalBaseUrl)
val rP = new RecordProcessor(mockDCH, mockFCW, props)
val cR = new ConsumerRecord[String, String]("_void", 1, 0, "1", incomingMessage)
rP.process(cR)
}
......
File added
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment