Unverified Commit e2d7a847 authored by Sebastian Schüpbach's avatar Sebastian Schüpbach
Browse files

general refactoring

parent 703508e8
Pipeline #14169 passed with stages
in 12 minutes and 6 seconds
...@@ -17,14 +17,14 @@ ...@@ -17,14 +17,14 @@
~ along with this program. If not, see <https://www.gnu.org/licenses/>. ~ along with this program. If not, see <https://www.gnu.org/licenses/>.
--> -->
<Configuration status="debug" name="media-converter" packages=""> <Configuration status="info" name="media-converter" packages="">
<Appenders> <Appenders>
<Console name="STDOUT" target="SYSTEM_OUT"> <Console name="STDOUT" target="SYSTEM_OUT">
<PatternLayout pattern="[%-5level] [%c{1}] %m%n"/> <PatternLayout pattern="[%-5level] [%c{1}] %m%n"/>
</Console> </Console>
</Appenders> </Appenders>
<Loggers> <Loggers>
<Root level="debug"> <Root level="info">
<AppenderRef ref="STDOUT"/> <AppenderRef ref="STDOUT"/>
</Root> </Root>
</Loggers> </Loggers>
......
...@@ -65,8 +65,8 @@ class DisseminationCopyHandler(audioDestPath: String, imageDestPath: String, vid ...@@ -65,8 +65,8 @@ class DisseminationCopyHandler(audioDestPath: String, imageDestPath: String, vid
val destFile = Paths.get(audioDestPath, destId + ".mp4") val destFile = Paths.get(audioDestPath, destId + ".mp4")
val snippetFile = Paths.get(audioDestPath, destId + "-intro." + Conversions.getFileTypeExtension(sourceFileType).get) val snippetFile = Paths.get(audioDestPath, destId + "-intro." + Conversions.getFileTypeExtension(sourceFileType).get)
writeData(data, tempFilePath) writeData(data, tempFilePath)
Transformations.audioToMp4(tempFilePath.toString, destFile.toString).get MediaTransformations.audioToMp4(tempFilePath.toString, destFile.toString).get
Transformations.createAudioSnippet(tempFilePath.toString, snippetFile.toString, audioSnippetDuration) MediaTransformations.createAudioSnippet(tempFilePath.toString, snippetFile.toString, audioSnippetDuration)
Files.delete(tempFilePath) Files.delete(tempFilePath)
destFile destFile
} }
...@@ -83,7 +83,7 @@ class DisseminationCopyHandler(audioDestPath: String, imageDestPath: String, vid ...@@ -83,7 +83,7 @@ class DisseminationCopyHandler(audioDestPath: String, imageDestPath: String, vid
val tempFilePath = Files.createTempFile("media-", "." + Conversions.getFileTypeExtension(sourceFileType).get) val tempFilePath = Files.createTempFile("media-", "." + Conversions.getFileTypeExtension(sourceFileType).get)
val destFile = Paths.get(imageDestPath, destId + ".jp2") val destFile = Paths.get(imageDestPath, destId + ".jp2")
writeData(data, tempFilePath) writeData(data, tempFilePath)
Transformations.imageToJp2(tempFilePath.toString, destFile.toString).get MediaTransformations.imageToJp2(tempFilePath.toString, destFile.toString).get
Files.delete(tempFilePath) Files.delete(tempFilePath)
destFile destFile
} }
......
...@@ -29,7 +29,7 @@ import scala.util.Try ...@@ -29,7 +29,7 @@ import scala.util.Try
/** /**
* Contains functions used to transform specific media files * Contains functions used to transform specific media files
*/ */
object Transformations extends Logging { object MediaTransformations extends Logging {
import sys.process._ import sys.process._
......
...@@ -44,12 +44,14 @@ class RecordProcessor(fileHandler: DisseminationCopyHandler, fedoraClientWrapper ...@@ -44,12 +44,14 @@ class RecordProcessor(fileHandler: DisseminationCopyHandler, fedoraClientWrapper
} }
def process(record: ConsumerRecord[String, String]): ProcessOutcome = { def process(record: ConsumerRecord[String, String]): ProcessOutcome = {
(for { val test = (for {
kafkaMsg <- BinaryResourceMetadata.build(record.value(), externalBaseUrl) kafkaMsg <- BinaryResourceMetadata.build(record.value(), externalBaseUrl)
fileWithMetadata <- fedoraClientWrapper.fetchBinaryResource(kafkaMsg.filePath) fileWithMetadata <- fedoraClientWrapper.fetchBinaryResource(kafkaMsg.filePath)
} yield createProcessResult(kafkaMsg.id, kafkaMsg.eventType, fileWithMetadata.fileType, fileWithMetadata.data)) } yield createProcessResult(kafkaMsg.id, kafkaMsg.eventType, fileWithMetadata.fileType, fileWithMetadata.data))
test
.recover { .recover {
case e: ResourceWithoutBinary => ProcessIgnore(record.key(), e.getMessage) case e: NoLocalBinary => ProcessIgnore(record.key(), e.getMessage)
case e: NoDigitalObject => ProcessIgnore(record.key(), e.getMessage)
case e: Exception => ProcessFailure(record.key(), UnknownFileType, "", e) case e: Exception => ProcessFailure(record.key(), UnknownFileType, "", e)
} }
}.get }.get
......
...@@ -22,7 +22,7 @@ package ch.memobase.models ...@@ -22,7 +22,7 @@ package ch.memobase.models
import ujson.{Str, Value} import ujson.{Str, Value}
import scala.collection.mutable.ArrayBuffer import scala.collection.mutable.ArrayBuffer
import scala.util.{Failure, Success, Try} import scala.util.{Success, Try}
/** /**
...@@ -48,45 +48,80 @@ object BinaryResourceMetadata { ...@@ -48,45 +48,80 @@ object BinaryResourceMetadata {
* @param externalBaseUrl Base URL of resource used outside of Fedora * @param externalBaseUrl Base URL of resource used outside of Fedora
* @return * @return
*/ */
def build(msg: String, externalBaseUrl: String): Try[BinaryResourceMetadata] = Try { def build(msg: String, externalBaseUrl: String): Try[BinaryResourceMetadata] = {
val jsonldGraph = ujson.read(msg).obj("@graph").arr val jsonldGraph = ujson.read(msg).obj("@graph").arr
buildKafkaMessage(jsonldGraph, externalBaseUrl) match { extractBinaryResourceMetadata(jsonldGraph, externalBaseUrl)
case Some(km) => km
case None => throw new ResourceWithoutBinary("Resource contains no binary object")
}
} }
private def setEventType(eventAsString: String): Event = eventAsString match { private def chooseEventType(eventAsString: String): Event = eventAsString match {
case "Create" => Create case "Create" => Create
case "Update" => Update case "Update" => Update
case "Delete" => Delete case "Delete" => Delete
case s => UnknownEvent(s)
} }
private def buildKafkaMessage(jsonldGraph: ArrayBuffer[Value], baseUrl: String): Option[BinaryResourceMetadata] = { //noinspection ScalaStyle
jsonldGraph.value private def extractBinaryResourceMetadata(jsonldGraph: ArrayBuffer[Value], baseUrl: String): Try[BinaryResourceMetadata] = Try {
.withFilter(v => isDigitalBinaryObject(v.obj, baseUrl)) val digitalObject = jsonldGraph.value
.map { o => .collectFirst { case v if isDigitalObject(v.obj) => v.obj }
digitalObject match {
case Some(obj) if isLocalRecord(obj, baseUrl) && isProcessableMimeType(obj) =>
getEventType(jsonldGraph) match {
case Some(UnknownEvent(e)) => throw new UnknownEventType(s"Event type `$e` not known")
case Some(eventType) =>
BinaryResourceMetadata( BinaryResourceMetadata(
o("@id").str.substring(s"$baseUrl/digital/".length).replaceFirst("/binary", ""), obj("@id").str.substring(s"$baseUrl/digital/".length),
o("@id").str, obj("locator").str,
o("hasMimeType").str, obj("hasMimeType").str,
setEventType(o("eventType").str)) eventType)
case None => throw new NoEventType
}
case Some(obj) if isLocalRecord(obj, baseUrl) => throw new UnmanageableMediaFileType("Media file type unknown")
case Some(_) => throw new NoLocalBinary
case None => throw new NoDigitalObject
}
}
private def isDigitalObject(obj: ujson.Obj): Boolean = {
hasKeyValue(obj, "type") {
_ == "digitalObject"
}
}
private def isRecord(obj: ujson.Obj): Boolean = {
hasKeyValue(obj, "@type") {
_ == "https://www.ica.org/standards/RiC/ontology#Record"
} }
.headOption
} }
private def isDigitalBinaryObject(obj: ujson.Obj, internalBaseUrl: String): Boolean = { private def isProcessableMimeType(obj: ujson.Obj): Boolean = {
isObjectWrapper(obj) { hasKeyValue(obj, "hasMimeType") {
id => id.startsWith(s"$internalBaseUrl/digital") && id.endsWith("/binary") value => Conversions.getMediaFileType(value).isDefined
} }
} }
private def isObjectWrapper(obj: ujson.Obj)(f: String => Boolean): Boolean = { private def isLocalRecord(obj: ujson.Obj, externalBaseUrl: String): Boolean = {
Try(obj.value("@id")) match { hasKeyValue(obj, "locator") {
case Success(id: Str) => f(id.value) value => value.startsWith(externalBaseUrl)
case Success(_) => false }
case Failure(_) => false
} }
private def hasKeyValue(obj: ujson.Obj, key: String)(valueFun: String => Boolean): Boolean = {
Try(obj.value(key)) match {
case Success(id: Str) => valueFun(id.value)
case _ => false
} }
}
private def getEventType(objList: ArrayBuffer[Value]): Option[Event] = {
objList
.collectFirst {
case v if isRecord(v.obj) && v.obj.contains("eventType") => chooseEventType(v.obj("eventType").str)
}
}
} }
...@@ -38,3 +38,8 @@ case object Update extends Event ...@@ -38,3 +38,8 @@ case object Update extends Event
* Equals a `delete` event type produced by Fedora * Equals a `delete` event type produced by Fedora
*/ */
case object Delete extends Event case object Delete extends Event
/**
* If event is unknown
*/
case class UnknownEvent(eventName: String) extends Event
...@@ -27,4 +27,11 @@ package ch.memobase.models ...@@ -27,4 +27,11 @@ package ch.memobase.models
//noinspection ScalaFileName //noinspection ScalaFileName
class UnmanageableMediaFileType(msg: String) extends Exception(msg) class UnmanageableMediaFileType(msg: String) extends Exception(msg)
class ResourceWithoutBinary(msg: String) extends Exception(msg) class NoDigitalObject extends Exception("No digital object found")
class NoLocalBinary extends Exception("No reference to local binary found")
class UnknownEventType(msg: String) extends Exception(msg)
class NoEventType extends Exception("No event type found")
...@@ -59,6 +59,11 @@ case object OgaFile extends AudioFileType ...@@ -59,6 +59,11 @@ case object OgaFile extends AudioFileType
*/ */
case object JpegFile extends ImageFileType case object JpegFile extends ImageFileType
/**
* Represents a PNG file
*/
case object PngFile extends ImageFileType
/** /**
* Represents a MPEG4 video file * Represents a MPEG4 video file
*/ */
...@@ -71,7 +76,8 @@ object Conversions { ...@@ -71,7 +76,8 @@ object Conversions {
private val fileTypeTuples: List[(MediaFileType, List[String], String)] = List( private val fileTypeTuples: List[(MediaFileType, List[String], String)] = List(
(Mp3File, List("audio/mpeg"), "mp3"), (Mp3File, List("audio/mpeg"), "mp3"),
(OgaFile, List("audio/ogg"), "oga"), (OgaFile, List("audio/ogg"), "oga"),
(JpegFile, List("image/jpg"), "jpg"), (JpegFile, List("image/jpeg"), "jpg"),
(PngFile, List("image/png"), "png"),
(VideoMpeg4File, List("video/mp4"), "mp4") (VideoMpeg4File, List("video/mp4"), "mp4")
// TODO: Other filetypes... // TODO: Other filetypes...
) )
......
...@@ -17,6 +17,8 @@ ...@@ -17,6 +17,8 @@
"lastModified": "2020-06-30T10:07:26.563Z", "lastModified": "2020-06-30T10:07:26.563Z",
"lastModifiedBy": "fedoraAdmin", "lastModifiedBy": "fedoraAdmin",
"contains": "https://memobase.ch/digital/BAZ-MEI_77466-1/binary", "contains": "https://memobase.ch/digital/BAZ-MEI_77466-1/binary",
"hasMimeType": "{{mimeType}}",
"locator": "{{locator}}",
"identifiedBy": [ "identifiedBy": [
"https://memobase.ch/digital/BAZ-MEI_77466-1#genidd1c6f2c6-99a9-407a-970a-1ec31a8e0292", "https://memobase.ch/digital/BAZ-MEI_77466-1#genidd1c6f2c6-99a9-407a-970a-1ec31a8e0292",
"https://memobase.ch/digital/BAZ-MEI_77466-1#genidada014e8-ead3-459f-a7ea-ac0aaf02b392" "https://memobase.ch/digital/BAZ-MEI_77466-1#genidada014e8-ead3-459f-a7ea-ac0aaf02b392"
...@@ -58,14 +60,8 @@ ...@@ -58,14 +60,8 @@
}, },
{ {
"@id": "https://memobase.ch/record/BAZ-MEI_77466", "@id": "https://memobase.ch/record/BAZ-MEI_77466",
"@type": [ "@type": "https://www.ica.org/standards/RiC/ontology#Record",
"fedora:Resource", "eventType": "{{eventType}}",
"https://www.ica.org/standards/RiC/ontology#Record",
"ldp:BasicContainer",
"ldp:Container",
"fedora:Container",
"ldp:RDFSource"
],
"fedora:created": { "fedora:created": {
"@type": "http://www.w3.org/2001/XMLSchema#dateTime", "@type": "http://www.w3.org/2001/XMLSchema#dateTime",
"@value": "2020-06-30T09:45:42.286Z" "@value": "2020-06-30T09:45:42.286Z"
......
...@@ -16,6 +16,8 @@ ...@@ -16,6 +16,8 @@
"createdBy": "fedoraAdmin", "createdBy": "fedoraAdmin",
"lastModified": "2020-06-30T10:07:26.563Z", "lastModified": "2020-06-30T10:07:26.563Z",
"lastModifiedBy": "fedoraAdmin", "lastModifiedBy": "fedoraAdmin",
"contains": "https://memobase.ch/digital/BAZ-MEI_77466-1/binary",
"hasMimeType": "{{mimeType}}",
"identifiedBy": [ "identifiedBy": [
"https://memobase.ch/digital/BAZ-MEI_77466-1#genidd1c6f2c6-99a9-407a-970a-1ec31a8e0292", "https://memobase.ch/digital/BAZ-MEI_77466-1#genidd1c6f2c6-99a9-407a-970a-1ec31a8e0292",
"https://memobase.ch/digital/BAZ-MEI_77466-1#genidada014e8-ead3-459f-a7ea-ac0aaf02b392" "https://memobase.ch/digital/BAZ-MEI_77466-1#genidada014e8-ead3-459f-a7ea-ac0aaf02b392"
...@@ -57,14 +59,8 @@ ...@@ -57,14 +59,8 @@
}, },
{ {
"@id": "https://memobase.ch/record/BAZ-MEI_77466", "@id": "https://memobase.ch/record/BAZ-MEI_77466",
"@type": [ "@type": "https://www.ica.org/standards/RiC/ontology#Record",
"fedora:Resource", "eventType": "{{eventType}}",
"https://www.ica.org/standards/RiC/ontology#Record",
"ldp:BasicContainer",
"ldp:Container",
"fedora:Container",
"ldp:RDFSource"
],
"fedora:created": { "fedora:created": {
"@type": "http://www.w3.org/2001/XMLSchema#dateTime", "@type": "http://www.w3.org/2001/XMLSchema#dateTime",
"@value": "2020-06-30T09:45:42.286Z" "@value": "2020-06-30T09:45:42.286Z"
......
...@@ -19,7 +19,7 @@ ...@@ -19,7 +19,7 @@
package ch.memobase package ch.memobase
import ch.memobase.models.BinaryResourceMetadata import ch.memobase.models.{BinaryResourceMetadata, NoLocalBinary, UnknownEventType, UnmanageableMediaFileType}
import org.scalatest.funsuite.AnyFunSuite import org.scalatest.funsuite.AnyFunSuite
import scala.io.Source import scala.io.Source
...@@ -28,17 +28,40 @@ class BinaryResourceMetadataTest extends AnyFunSuite { ...@@ -28,17 +28,40 @@ class BinaryResourceMetadataTest extends AnyFunSuite {
val externalBaseUrl = "https://memobase.ch" val externalBaseUrl = "https://memobase.ch"
private def loadMessage: String = { private def loadMessageWithBinaryResource(eventType: String, mimeType: String, locator: String): String = {
val file = Source.fromFile("src/test/resources/incoming_message_with_binary.json") val file = Source.fromFile("src/test/resources/incoming_message_with_binary.json")
val result = file.mkString val result = file.mkString
.replaceAll(raw"\{\{eventType\}\}", "Create") .replaceAll(raw"\{\{eventType\}\}", eventType)
.replaceAll(raw"\{\{mimeType\}\}", "image/jpeg") .replaceAll(raw"\{\{mimeType\}\}", mimeType)
.replaceAll(raw"\{\{locator\}\}", locator)
file.close() file.close()
result result
} }
test("the value of the id field of a KafkaMessage should match the id of the parsed object") { test("the value of the id field of a KafkaMessage should match the id of the parsed object") {
val km = BinaryResourceMetadata.build(loadMessage, externalBaseUrl) val km = BinaryResourceMetadata.build(loadMessageWithBinaryResource("Create", "image/jpeg", "https://memobase.ch/digital/BAZ-MEI_77466-1/binary"), externalBaseUrl)
assert(km.get.id == "BAZ-MEI_77466-1") assert(km.isSuccess)
}
test("a reference to a non-local binary should throw a NoLocalBinary exception") {
assertThrows[NoLocalBinary] {
BinaryResourceMetadata.build(loadMessageWithBinaryResource("Create", "image/jpeg", "https://example.com"), externalBaseUrl).get
}
}
test("a unmanageable mime type should throw a UnmanageableMediaFileType exception") {
assertThrows[UnmanageableMediaFileType] {
BinaryResourceMetadata.build(loadMessageWithBinaryResource("Create",
"application/pdf",
"https://memobase.ch/digital/BAZ-MEI_77466-1/binary"), externalBaseUrl).get
}
}
test("a unknown event type should throw a UnknownEventType exception") {
assertThrows[UnknownEventType] {
BinaryResourceMetadata.build(loadMessageWithBinaryResource("Upload",
"image/jpeg",
"https://memobase.ch/digital/BAZ-MEI_77466-1/binary"), externalBaseUrl).get
}
} }
} }
...@@ -25,6 +25,7 @@ import java.nio.file.{Files, Path, Paths} ...@@ -25,6 +25,7 @@ import java.nio.file.{Files, Path, Paths}
import ch.memobase.models.{JpegFile, MediaFileType, Mp3File, VideoMpeg4File} import ch.memobase.models.{JpegFile, MediaFileType, Mp3File, VideoMpeg4File}
import org.scalatest.funsuite.AnyFunSuite import org.scalatest.funsuite.AnyFunSuite
import org.scalatest.{Assertion, BeforeAndAfter} import org.scalatest.{Assertion, BeforeAndAfter}
import TestUtilities._
import scala.util.Try import scala.util.Try
...@@ -77,27 +78,33 @@ class DisseminationCopyHandlerTest extends AnyFunSuite with BeforeAndAfter { ...@@ -77,27 +78,33 @@ class DisseminationCopyHandlerTest extends AnyFunSuite with BeforeAndAfter {
* ATTENTION: Requires that ffmpeg is properly installed! * ATTENTION: Requires that ffmpeg is properly installed!
*/ */
test("calling the copyAudio function should create temporary file") { test("calling the copyAudio function should create temporary file") {
runWithFFmpeg {
val f = fixture val f = fixture
testCopy(f.resPath, "sample.mp3", "test.mp4", Mp3File, f.fileHandler.createAudioCopy) testCopy(f.resPath, "sample.mp3", "test.mp4", Mp3File, f.fileHandler.createAudioCopy)
deleteFiles("src/test/resources/test.mp4", "src/test/resources/test-intro.mp3") deleteFiles("src/test/resources/test.mp4", "src/test/resources/test-intro.mp3")
} }
}
/** /**
* ATTENTION: Requires that Kakadu and imagemagick are properly installed! * ATTENTION: Requires that Kakadu and imagemagick are properly installed!
*/ */
test("calling the copyImage function should create temporary file") { test("calling the copyImage function should create temporary file") {
runWithKakaduAndIM {
val f = fixture val f = fixture
testCopy(f.resPath, "sample.jpg", "test.jp2", JpegFile, f.fileHandler.createImageCopy) testCopy(f.resPath, "sample.jpg", "test.jp2", JpegFile, f.fileHandler.createImageCopy)
deleteFiles("src/test/resources/test.jp2") deleteFiles("src/test/resources/test.jp2")
} }
}
/** /**
* ATTENTION: Requires that ffmpeg is properly installed! * ATTENTION: Requires that ffmpeg is properly installed!
*/ */
test("calling the copyVideo function should create temporary file") { test("calling the copyVideo function should create temporary file") {
runWithFFmpeg {
val f = fixture val f = fixture
testCopy(f.resPath, "sample.mp4", "test.mp4", VideoMpeg4File, f.fileHandler.createVideoCopy) testCopy(f.resPath, "sample.mp4", "test.mp4", VideoMpeg4File, f.fileHandler.createVideoCopy)
deleteFiles("src/test/resources/test.mp4") deleteFiles("src/test/resources/test.mp4")
} }
}
} }
...@@ -22,46 +22,35 @@ package ch.memobase ...@@ -22,46 +22,35 @@ package ch.memobase
import java.nio.file.{Files, Paths} import java.nio.file.{Files, Paths}
import org.scalatest.funsuite.AnyFunSuite import org.scalatest.funsuite.AnyFunSuite
import TestUtilities._
class TransformationsTest extends AnyFunSuite { class MediaTransformationsTest extends AnyFunSuite {
private def appExists(name: String): Boolean = {
import sys.process._
(name !) == 0
}
test("Sample mp3 should be transformed correctly") { test("Sample mp3 should be transformed correctly") {
if (appExists("ffmpeg -version")) { runWithFFmpeg {
val outFile = Files.createTempFile(Paths.get("src/test/resources"), "test-", ".mp4") val outFile = Files.createTempFile(Paths.get("src/test/resources"), "test-", ".mp4")
val res = Transformations.audioToMp4("src/test/resources/sample.mp3", outFile.toString) val res = MediaTransformations.audioToMp4("src/test/resources/sample.mp3", outFile.toString)
outFile.toFile.delete() outFile.toFile.delete()
assert(res.get == outFile.toString) assert(res.get == outFile.toString)
} else {
println("No ffmpeg binary found in $PATH")
} }
} }
test("Conversion of nonexistent mp3 should abort with error") { test("Conversion of nonexistent mp3 should abort with error") {
if (appExists("ffmpeg -version")) { runWithFFmpeg {
val outFile = Files.createTempFile(Paths.get("src/test/resources"), "test-", ".mp4") val outFile = Files.createTempFile(Paths.get("src/test/resources"), "test-", ".mp4")
val res = Transformations.audioToMp4("src/test/resources/null.mp3", outFile.toString) val res = MediaTransformations.audioToMp4("src/test/resources/null.mp3", outFile.toString)
outFile.toFile.delete() outFile.toFile.delete()
assert(res.isFailure) assert(res.isFailure)
} else {
println("No ffmpeg binary found in $PATH")
} }
} }
test("Sample jpeg should be transformed into jp2") { test("Sample jpeg should be transformed into jp2") {
if (appExists("kdu_compress -v") && appExists("convert -version")) { runWithKakaduAndIM {
val outFile = Files.createTempFile(Paths.get("src/test/resources"), "test-", ".jp2") val outFile = Files.createTempFile(Paths.get("src/test/resources"), "test-", ".jp2")
val res = Transformations.imageToJp2("src/test/resources/sample.jpg", outFile.toString) val res = MediaTransformations.imageToJp2("src/test/resources/sample.jpg", outFile.toString)
outFile.toFile.delete() outFile.toFile.delete()
assert(res.isSuccess, res) assert(res.isSuccess, res)
} else if (!appExists("kdu_compress -v")) }
println("No kdu_compress binary found in $PATH")
else if (!appExists("convert -version"))
println("No convert binary found in $PATH")
}