Skip to content

GitLab

Unverified Commit 4b8ca9ee authored by Sebastian Schüpbach's avatar Sebastian Schüpbach
Browse files

remove code for processing delete events

parent 0190b4c8
Pipeline #21272 passed with stages
in 8 minutes and 29 seconds
Hide whitespace changes
Inline Side-by-side
src/main/scala/ch/memobase/DisseminationCopyHandler.scala
View file @ 4b8ca9ee
......@@ -65,7 +65,6 @@ class DisseminationCopyHandler(audioSnippetDuration: Int) extends Logging {
}
}
/**
* Creates dissemination copy of audio file
*
......@@ -117,38 +116,4 @@ class DisseminationCopyHandler(audioSnippetDuration: Int) extends Logging {
writeData(data, destFileAsPath)
copyRemoved
}
/**
* Deletes dissemination copy of audio file
*
* @param destFile Path to dissemination copy
* @return true if copy and snippet were deleted successfully, false otherwise
*/
def deleteAudioCopy(destFile: String): Try[Boolean] =
Try {
Paths.get(destFile).toFile.delete()
}
/**
* Deletes dissemination copy of image file
*
* @param destFile Path to dissemination copy
* @return true if copy was deleted successfully, false otherwise
*/
def deleteImageCopy(destFile: String): Try[Boolean] =
Try {
Paths.get(destFile).toFile.delete()
}
/**
* Deletes dissemination copy of video file
*
* @param destFile Path to dissemination copy
* @return true if copy was deleted successfully, false otherwise
*/
def deleteVideoCopy(destFile: String): Try[Boolean] =
Try {
Paths.get(destFile).toFile.delete()
}
}
src/main/scala/ch/memobase/RecordProcessor.scala
View file @ 4b8ca9ee
......@@ -60,8 +60,6 @@ class RecordProcessor(fileHandler: DisseminationCopyHandler,
private def handleBinaryResource(binaryResource: BinaryResourceMetadata, recordKey: String): List[ProcessOutcome] = {
fetchBinaryResource(binaryResource.filePath) match {
case Success(_) if binaryResource.eventType == Delete =>
deleteResource(binaryResource.id, binaryResource.mimeType, binaryResource.resource)
case Success(tempFilePath) =>
createResource(
binaryResource.id,
......@@ -79,34 +77,6 @@ class RecordProcessor(fileHandler: DisseminationCopyHandler,
case Failure(ex) => ProcessFatal(id, resource, s"Creation of file $destFile failed", ex)
})
private def deleteOutcome(res: Try[Boolean], id: String, resource: MemobaseResource, destFile: String): List[ProcessOutcome] = List(res match {
case Success(true) => ProcessSuccess(id, resource, s"Deletion of file $destFile successful")
case Success(false) => ProcessSuccess(id, resource, s"No deletion of file $destFile because object does not exist")
case Failure(ex) => ProcessFatal(id, resource, s"Deletion of file $destFile failed", ex)
})
private def deleteResource(id: String,
mimeType: MimeType,
resource: MemobaseResource): List[ProcessOutcome] = mimeType match {
case _: AudioFile =>
List((audioFilePath(id), DigitalObject), (audioSnippetPath(id), AudioSnippet))
.map(obj => (fileHandler.deleteAudioCopy(obj._1), obj._2, obj._1))
.flatMap(x => deleteOutcome(x._1, id, x._2, x._3))
case mT: VideoFile =>
val destFile = videoFilePath(id, mT)
val res = fileHandler.deleteVideoCopy(destFile)
deleteOutcome(res, id, DigitalObject, destFile)
case mT: ImageFile if resource == DigitalObject =>
val destFile = imageFilePath(id, mT)
val res = fileHandler.deleteImageCopy(destFile)
deleteOutcome(res, id, DigitalObject, destFile)
case mT: ImageFile if resource == Thumbnail =>
val destFile = videoPosterPath(id, mT)
val res = fileHandler.deleteImageCopy(destFile)
deleteOutcome(res, id, Thumbnail, destFile)
}
private def createResource(id: String,
mimeType: MimeType,
resource: MemobaseResource,
......@@ -144,5 +114,4 @@ class RecordProcessor(fileHandler: DisseminationCopyHandler,
.foreach(baos.write)
baos
}
}
}
\ No newline at end of file
src/main/scala/ch/memobase/RecordUtils.scala
View file @ 4b8ca9ee
......@@ -26,18 +26,10 @@ import scala.collection.mutable.ArrayBuffer
import scala.util.{Success, Try}
trait RecordUtils {
protected def getJsonldGraph(msg: String): ArrayBuffer[Value] = {
ujson.read(msg).obj("@graph").arr
}
private def chooseEventType(eventAsString: String): Event = eventAsString match {
case "Create" | "CREATE" => Create
case "Update" | "UPDATE" => Update
case "Delete" | "DELETE" => Delete
case s => UnknownEvent(s)
}
protected def isDigitalObject(obj: ujson.Obj): Boolean = {
hasKeyValue(obj, "type") {
MemobaseResource(_) == DigitalObject
......@@ -68,18 +60,4 @@ trait RecordUtils {
case _ => false
}
}
protected def getEventType(objList: ArrayBuffer[Value]): Option[Event] = {
objList
.collectFirst {
case v if isRecord(v.obj) && v.obj.contains("eventType") => chooseEventType(v.obj("eventType").str)
}
}
private def isRecord(obj: ujson.Obj): Boolean = {
hasKeyValue(obj, "@type") {
_ == "https://www.ica.org/standards/RiC/ontology#Record"
}
}
}
src/main/scala/ch/memobase/models/BinaryResourceMetadata.scala
View file @ 4b8ca9ee
......@@ -29,17 +29,15 @@ import scala.util.Try
/**
* Essential information on a binary file residing in Fedora
*
* @param id Identifier of the binary file
* @param filePath File path (URL) to resource
* @param mimeType MIME type
* @param resource Type of instantiation
* @param eventType Action to be performed on the copy of the file
* @param id Identifier of the binary file
* @param filePath File path (URL) to resource
* @param mimeType MIME type
* @param resource Type of instantiation
*/
case class BinaryResourceMetadata(id: String,
filePath: String,
mimeType: MimeType,
resource: MemobaseResource,
eventType: Event) {
resource: MemobaseResource) {
}
object BinaryResourceMetadata extends RecordUtils {
......@@ -84,22 +82,16 @@ object BinaryResourceMetadata extends RecordUtils {
Try(
v.obj match {
case v if isLocalRecord(v) && isProcessableMimeType(v) =>
getEventType(jsonldGraph) match {
case Some(UnknownEvent(e)) => throw new UnknownEventType(s"Event type `$e` for $id not known")
case Some(eventType) =>
val instantiation = MemobaseResource(v("type").str)
BinaryResourceMetadata(
v("@id").str.substring(s"$baseUrl/digital/".length - 1),
buildDistributorUrl(v("@id").str, baseUrl, distributorHost, instantiation),
if (instantiation == Thumbnail) {
JpegFile
} else {
Conversions.getMediaFileType(v("hasMimeType").str).get
},
instantiation,
eventType)
case None => throw new NoEventType(id)
}
val instantiation = MemobaseResource(v("type").str)
BinaryResourceMetadata(
v("@id").str.substring(s"$baseUrl/digital/".length - 1),
buildDistributorUrl(v("@id").str, baseUrl, distributorHost, instantiation),
if (instantiation == Thumbnail) {
JpegFile
} else {
Conversions.getMediaFileType(v("hasMimeType").str).get
},
instantiation)
case v if isLocalRecord(v) =>
val resource = MemobaseResource(v("type").str)
throw new UnmanageableMediaFileType(s"Media file type for $id unknown", resource)
......@@ -108,7 +100,6 @@ object BinaryResourceMetadata extends RecordUtils {
throw new NoLocalBinary(id, resource)
}
)
}
}.toList
}
src/main/scala/ch/memobase/models/Events.scala deleted 100644 → 0
View file @ 0190b4c8
/*
* Media Converter
* Extracts media files from Fedora repository
* Copyright (C) 2021 Memoriav
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package ch.memobase.models
/**
* Represents a type of an event produced by Fedora
*/
sealed trait Event
/**
* Equals a `create` event type produced by Fedora
*/
case object Create extends Event
/**
* Equals an `update` event type produced by Fedora
*/
case object Update extends Event
/**
* Equals a `delete` event type produced by Fedora
*/
case object Delete extends Event
/**
* If event is unknown
*/
case class UnknownEvent(eventName: String) extends Event
src/main/scala/ch/memobase/models/Exceptions.scala
View file @ 4b8ca9ee
......@@ -31,7 +31,3 @@ class NoDigitalObject extends Exception("No digital object found")
class NoLocalBinary(id: String, val resource: MemobaseResource) extends Exception(s"No reference to local binary in $id found")
class UnknownEventType(msg: String) extends Exception(msg)
class NoEventType(id: String) extends Exception(s"No event type for $id found")
src/test/scala/ch/memobase/BinaryResourceMetadataTest.scala
View file @ 4b8ca9ee
......@@ -19,7 +19,7 @@
package ch.memobase
import ch.memobase.models.{BinaryResourceMetadata, NoLocalBinary, UnknownEventType, UnmanageableMediaFileType}
import ch.memobase.models.{BinaryResourceMetadata, NoLocalBinary, UnmanageableMediaFileType}
import org.scalatest.funsuite.AnyFunSuite
import scala.io.Source
......@@ -28,10 +28,9 @@ class BinaryResourceMetadataTest extends AnyFunSuite {
val externalBaseUrl = "https://memobase.ch"
private def loadMessageWithBinaryResource(eventType: String, mimeType: String, locator: String): String = {
private def loadMessageWithBinaryResource(mimeType: String, locator: String): String = {
val file = Source.fromFile("src/test/resources/incoming_message_with_binary.json")
val result = file.mkString
.replaceAll(raw"\{\{eventType\}\}", eventType)
.replaceAll(raw"\{\{mimeType\}\}", mimeType)
.replaceAll(raw"\{\{locator\}\}", locator)
file.close()
......@@ -39,13 +38,13 @@ class BinaryResourceMetadataTest extends AnyFunSuite {
}
test("the value of the id field of a KafkaMessage should match the id of the parsed object") {
val km = BinaryResourceMetadata.build(loadMessageWithBinaryResource("Create", "image/png", "sftp:/BAZ-MEI_77466-1/binary"), externalBaseUrl, "mb-wf2")
val km = BinaryResourceMetadata.build(loadMessageWithBinaryResource("image/png", "sftp:/BAZ-MEI_77466-1/binary"), externalBaseUrl, "mb-wf2")
assert(km.head.isSuccess)
}
test("a reference to a non-local binary should throw a NoLocalBinary exception") {
assertThrows[NoLocalBinary] {
BinaryResourceMetadata.build(loadMessageWithBinaryResource("Create", "image/jpeg", "https://example.com"), externalBaseUrl, "mb-wf2")
BinaryResourceMetadata.build(loadMessageWithBinaryResource("image/jpeg", "https://example.com"), externalBaseUrl, "mb-wf2")
.head
.get
}
......@@ -53,18 +52,7 @@ class BinaryResourceMetadataTest extends AnyFunSuite {
test("a unmanageable mime type should throw a UnmanageableMediaFileType exception") {
assertThrows[UnmanageableMediaFileType] {
BinaryResourceMetadata.build(loadMessageWithBinaryResource("Create",
"application/pdf",
"sftp:/BAZ-MEI_77466-1/binary"), externalBaseUrl, "mb-wf2")
.head
.get
}
}
test("a unknown event type should throw a UnknownEventType exception") {
assertThrows[UnknownEventType] {
BinaryResourceMetadata.build(loadMessageWithBinaryResource("Upload",
"image/jpeg",
BinaryResourceMetadata.build(loadMessageWithBinaryResource("application/pdf",
"sftp:/BAZ-MEI_77466-1/binary"), externalBaseUrl, "mb-wf2")
.head
.get
......
src/test/scala/ch/memobase/DisseminationCopyHandlerTest.scala
View file @ 4b8ca9ee
......@@ -88,16 +88,6 @@ class DisseminationCopyHandlerTest extends AnyFunSuite with BeforeAndAfter {
}
test("calling the deleteAudio function should delete temporary audio file") {
val f = fixture
val testFile = Files.createFile(Paths.get(f.resPath, "test.mp4"))
val testSnippetFile = Files.createFile(Paths.get(f.resPath, "test-intro.mp3"))
f.fileHandler.deleteAudioCopy(testFile.toString)
f.fileHandler.deleteAudioCopy(testSnippetFile.toString)
assert(!testFile.toFile.exists())
assert(!testSnippetFile.toFile.exists())
}
/**
* ATTENTION: Requires that ffmpeg is properly installed!
*/
......
src/test/scala/ch/memobase/RecordProcessorTest.scala
View file @ 4b8ca9ee
......@@ -28,20 +28,14 @@ import scala.io.Source
class RecordProcessorTest extends AnyFunSuite with MockFactory {
def createIncomingMessage(eventType: Event, mimeType: MimeType, hasBinaryResource: Boolean = true): (String, ByteArrayOutputStream) = {
def createIncomingMessage(mimeType: MimeType, hasBinaryResource: Boolean = true): (String, ByteArrayOutputStream) = {
if (hasBinaryResource) {
val eT = eventType match {
case Create => "Create"
case Delete => "Delete"
case Update => "Update"
}
val mT = mimeType match {
case JpegFile => "image/jpeg"
case Mp3File => "audio/mpeg"
case VideoMpeg4File => "video/mpeg"
}
replaceTokensInIncomingMessage(eT, mT)
replaceTokensInIncomingMessage(mT)
} else {
val is = Source.fromFile("src/test/resources/incoming_message_without_binary.json")
val incomingMessage = is.mkString
......@@ -51,11 +45,10 @@ class RecordProcessorTest extends AnyFunSuite with MockFactory {
}
}
private def replaceTokensInIncomingMessage(eventType: String, mimeType: String, locator: String = "https://memobase.ch/digital/BAZ-MEI_77466-1/binary") = {
private def replaceTokensInIncomingMessage(mimeType: String, locator: String = "https://memobase.ch/digital/BAZ-MEI_77466-1/binary") = {
val is = Source.fromFile("src/test/resources/incoming_message_with_binary.json")
val incomingMessage = is.mkString
.replaceAll(raw"\{\{mimeType\}\}", mimeType)
.replaceAll(raw"\{\{eventType\}\}", eventType)
.replaceAll(raw"\{\{locator\}\}", locator)
val baos = copyIncomingMessage(incomingMessage)
is.close
......@@ -68,46 +61,4 @@ class RecordProcessorTest extends AnyFunSuite with MockFactory {
baos.write(incomingMessageAsBytes, 0, incomingMessageAsBytes.length)
baos
}
/* private val fixture = new {
val externalBaseUrl = "https://memobase.ch"
val resourceId = "BAZ-MEI_77466-1"
//val pathToResource = s"$externalBaseUrl/digital/$resourceId/binary"
//val mockDisseminationCopyHandler: DisseminationCopyHandler = mock[DisseminationCopyHandler]
} */
/* test("an object of mimeType image/jpeg and eventType Create should trigger copyImage") {
val f = fixture
val mockDCH = f.mockDisseminationCopyHandler
val mockFCW = f.mockFedoraClientWrapper
val (incomingMessage, baos) = createIncomingMessage(Create, JpegFile)
(mockFCW.fetchBinaryResource _).expects(f.pathToResource).returning(Success(baos))
(mockDCH.createImageCopy _).expects(baos, f.resourceId, JpegFile)
val rP = new RecordProcessor(mockDCH, mockFCW, f.externalBaseUrl)
val cR = new ConsumerRecord[String, String]("_void", 1, 0, "1", incomingMessage)
rP.process(cR)
}
test("an object of mimeType audio/mpeg and eventType Delete should trigger deleteAudio") {
val f = fixture
val mockDCH = f.mockDisseminationCopyHandler
val mockFCW = f.mockFedoraClientWrapper
val (incomingMessage, baos) = createIncomingMessage(Delete, Mp3File)
(mockFCW.fetchBinaryResource _).expects(f.pathToResource).returning(Success(baos))
(mockDCH.deleteAudioCopy _).expects(f.resourceId)
val rP = new RecordProcessor(mockDCH, mockFCW, f.externalBaseUrl)
val cR = new ConsumerRecord[String, String]("_void", 1, 0, "1", incomingMessage)
rP.process(cR)
} */
/* test("a message which does not refer to a binary file should be ignored") {
val f = fixture
val mockDCH = f.mockDisseminationCopyHandler
val (incomingMessage, _) = createIncomingMessage(Delete, Mp3File, hasBinaryResource = false)
val props = new Properties()
props.setProperty("externalBaseUrl", f.externalBaseUrl)
val rP = new RecordProcessor(mockDCH, mockFCW, props)
val cR = new ConsumerRecord[String, String]("_void", 1, 0, "1", incomingMessage)
rP.process(cR)
}*/
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment