Skip to content
GitLab
Projects
Groups
Snippets
Help
Loading...
Help
What's new
10
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Open sidebar
memoriav
M
Memobase 2020
S
services
postprocessing
Media Converter
Commits
4b8ca9ee
Unverified
Commit
4b8ca9ee
authored
Feb 05, 2021
by
Sebastian Schüpbach
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
remove code for processing delete events
parent
0190b4c8
Pipeline
#21272
passed with stages
in 8 minutes and 29 seconds
Changes
9
Pipelines
2
Hide whitespace changes
Inline
Side-by-side
Showing
9 changed files
with
24 additions
and
241 deletions
+24
-241
src/main/scala/ch/memobase/DisseminationCopyHandler.scala
src/main/scala/ch/memobase/DisseminationCopyHandler.scala
+0
-35
src/main/scala/ch/memobase/RecordProcessor.scala
src/main/scala/ch/memobase/RecordProcessor.scala
+1
-32
src/main/scala/ch/memobase/RecordUtils.scala
src/main/scala/ch/memobase/RecordUtils.scala
+0
-22
src/main/scala/ch/memobase/models/BinaryResourceMetadata.scala
...ain/scala/ch/memobase/models/BinaryResourceMetadata.scala
+15
-24
src/main/scala/ch/memobase/models/Events.scala
src/main/scala/ch/memobase/models/Events.scala
+0
-45
src/main/scala/ch/memobase/models/Exceptions.scala
src/main/scala/ch/memobase/models/Exceptions.scala
+0
-4
src/test/scala/ch/memobase/BinaryResourceMetadataTest.scala
src/test/scala/ch/memobase/BinaryResourceMetadataTest.scala
+5
-17
src/test/scala/ch/memobase/DisseminationCopyHandlerTest.scala
...test/scala/ch/memobase/DisseminationCopyHandlerTest.scala
+0
-10
src/test/scala/ch/memobase/RecordProcessorTest.scala
src/test/scala/ch/memobase/RecordProcessorTest.scala
+3
-52
No files found.
src/main/scala/ch/memobase/DisseminationCopyHandler.scala
View file @
4b8ca9ee
...
...
@@ -65,7 +65,6 @@ class DisseminationCopyHandler(audioSnippetDuration: Int) extends Logging {
}
}
/**
* Creates dissemination copy of audio file
*
...
...
@@ -117,38 +116,4 @@ class DisseminationCopyHandler(audioSnippetDuration: Int) extends Logging {
writeData
(
data
,
destFileAsPath
)
copyRemoved
}
/**
* Deletes dissemination copy of audio file
*
* @param destFile Path to dissemination copy
* @return true if copy and snippet were deleted successfully, false otherwise
*/
def
deleteAudioCopy
(
destFile
:
String
)
:
Try
[
Boolean
]
=
Try
{
Paths
.
get
(
destFile
).
toFile
.
delete
()
}
/**
* Deletes dissemination copy of image file
*
* @param destFile Path to dissemination copy
* @return true if copy was deleted successfully, false otherwise
*/
def
deleteImageCopy
(
destFile
:
String
)
:
Try
[
Boolean
]
=
Try
{
Paths
.
get
(
destFile
).
toFile
.
delete
()
}
/**
* Deletes dissemination copy of video file
*
* @param destFile Path to dissemination copy
* @return true if copy was deleted successfully, false otherwise
*/
def
deleteVideoCopy
(
destFile
:
String
)
:
Try
[
Boolean
]
=
Try
{
Paths
.
get
(
destFile
).
toFile
.
delete
()
}
}
src/main/scala/ch/memobase/RecordProcessor.scala
View file @
4b8ca9ee
...
...
@@ -60,8 +60,6 @@ class RecordProcessor(fileHandler: DisseminationCopyHandler,
private
def
handleBinaryResource
(
binaryResource
:
BinaryResourceMetadata
,
recordKey
:
String
)
:
List
[
ProcessOutcome
]
=
{
fetchBinaryResource
(
binaryResource
.
filePath
)
match
{
case
Success
(
_
)
if
binaryResource
.
eventType
==
Delete
=>
deleteResource
(
binaryResource
.
id
,
binaryResource
.
mimeType
,
binaryResource
.
resource
)
case
Success
(
tempFilePath
)
=>
createResource
(
binaryResource
.
id
,
...
...
@@ -79,34 +77,6 @@ class RecordProcessor(fileHandler: DisseminationCopyHandler,
case
Failure
(
ex
)
=>
ProcessFatal
(
id
,
resource
,
s
"Creation of file $destFile failed"
,
ex
)
})
private
def
deleteOutcome
(
res
:
Try
[
Boolean
],
id
:
String
,
resource
:
MemobaseResource
,
destFile
:
String
)
:
List
[
ProcessOutcome
]
=
List
(
res
match
{
case
Success
(
true
)
=>
ProcessSuccess
(
id
,
resource
,
s
"Deletion of file $destFile successful"
)
case
Success
(
false
)
=>
ProcessSuccess
(
id
,
resource
,
s
"No deletion of file $destFile because object does not exist"
)
case
Failure
(
ex
)
=>
ProcessFatal
(
id
,
resource
,
s
"Deletion of file $destFile failed"
,
ex
)
})
private
def
deleteResource
(
id
:
String
,
mimeType
:
MimeType
,
resource
:
MemobaseResource
)
:
List
[
ProcessOutcome
]
=
mimeType
match
{
case
_:
AudioFile
=>
List
((
audioFilePath
(
id
),
DigitalObject
),
(
audioSnippetPath
(
id
),
AudioSnippet
))
.
map
(
obj
=>
(
fileHandler
.
deleteAudioCopy
(
obj
.
_1
),
obj
.
_2
,
obj
.
_1
))
.
flatMap
(
x
=>
deleteOutcome
(
x
.
_1
,
id
,
x
.
_2
,
x
.
_3
))
case
mT
:
VideoFile
=>
val
destFile
=
videoFilePath
(
id
,
mT
)
val
res
=
fileHandler
.
deleteVideoCopy
(
destFile
)
deleteOutcome
(
res
,
id
,
DigitalObject
,
destFile
)
case
mT
:
ImageFile
if
resource
=
=
DigitalObject
=>
val
destFile
=
imageFilePath
(
id
,
mT
)
val
res
=
fileHandler
.
deleteImageCopy
(
destFile
)
deleteOutcome
(
res
,
id
,
DigitalObject
,
destFile
)
case
mT
:
ImageFile
if
resource
=
=
Thumbnail
=>
val
destFile
=
videoPosterPath
(
id
,
mT
)
val
res
=
fileHandler
.
deleteImageCopy
(
destFile
)
deleteOutcome
(
res
,
id
,
Thumbnail
,
destFile
)
}
private
def
createResource
(
id
:
String
,
mimeType
:
MimeType
,
resource
:
MemobaseResource
,
...
...
@@ -144,5 +114,4 @@ class RecordProcessor(fileHandler: DisseminationCopyHandler,
.
foreach
(
baos
.
write
)
baos
}
}
}
\ No newline at end of file
src/main/scala/ch/memobase/RecordUtils.scala
View file @
4b8ca9ee
...
...
@@ -26,18 +26,10 @@ import scala.collection.mutable.ArrayBuffer
import
scala.util.
{
Success
,
Try
}
trait
RecordUtils
{
protected
def
getJsonldGraph
(
msg
:
String
)
:
ArrayBuffer
[
Value
]
=
{
ujson
.
read
(
msg
).
obj
(
"@graph"
).
arr
}
private
def
chooseEventType
(
eventAsString
:
String
)
:
Event
=
eventAsString
match
{
case
"Create"
|
"CREATE"
=>
Create
case
"Update"
|
"UPDATE"
=>
Update
case
"Delete"
|
"DELETE"
=>
Delete
case
s
=>
UnknownEvent
(
s
)
}
protected
def
isDigitalObject
(
obj
:
ujson.Obj
)
:
Boolean
=
{
hasKeyValue
(
obj
,
"type"
)
{
MemobaseResource
(
_
)
==
DigitalObject
...
...
@@ -68,18 +60,4 @@ trait RecordUtils {
case
_
=>
false
}
}
protected
def
getEventType
(
objList
:
ArrayBuffer
[
Value
])
:
Option
[
Event
]
=
{
objList
.
collectFirst
{
case
v
if
isRecord
(
v
.
obj
)
&&
v
.
obj
.
contains
(
"eventType"
)
=>
chooseEventType
(
v
.
obj
(
"eventType"
).
str
)
}
}
private
def
isRecord
(
obj
:
ujson.Obj
)
:
Boolean
=
{
hasKeyValue
(
obj
,
"@type"
)
{
_
==
"https://www.ica.org/standards/RiC/ontology#Record"
}
}
}
src/main/scala/ch/memobase/models/BinaryResourceMetadata.scala
View file @
4b8ca9ee
...
...
@@ -29,17 +29,15 @@ import scala.util.Try
/**
* Essential information on a binary file residing in Fedora
*
* @param id Identifier of the binary file
* @param filePath File path (URL) to resource
* @param mimeType MIME type
* @param resource Type of instantiation
* @param eventType Action to be performed on the copy of the file
* @param id Identifier of the binary file
* @param filePath File path (URL) to resource
* @param mimeType MIME type
* @param resource Type of instantiation
*/
case
class
BinaryResourceMetadata
(
id
:
String
,
filePath
:
String
,
mimeType
:
MimeType
,
resource
:
MemobaseResource
,
eventType
:
Event
)
{
resource
:
MemobaseResource
)
{
}
object
BinaryResourceMetadata
extends
RecordUtils
{
...
...
@@ -84,22 +82,16 @@ object BinaryResourceMetadata extends RecordUtils {
Try
(
v
.
obj
match
{
case
v
if
isLocalRecord
(
v
)
&&
isProcessableMimeType
(
v
)
=>
getEventType
(
jsonldGraph
)
match
{
case
Some
(
UnknownEvent
(
e
))
=>
throw
new
UnknownEventType
(
s
"Event type `$e` for $id not known"
)
case
Some
(
eventType
)
=>
val
instantiation
=
MemobaseResource
(
v
(
"type"
).
str
)
BinaryResourceMetadata
(
v
(
"@id"
).
str
.
substring
(
s
"$baseUrl/digital/"
.
length
-
1
),
buildDistributorUrl
(
v
(
"@id"
).
str
,
baseUrl
,
distributorHost
,
instantiation
),
if
(
instantiation
==
Thumbnail
)
{
JpegFile
}
else
{
Conversions
.
getMediaFileType
(
v
(
"hasMimeType"
).
str
).
get
},
instantiation
,
eventType
)
case
None
=>
throw
new
NoEventType
(
id
)
}
val
instantiation
=
MemobaseResource
(
v
(
"type"
).
str
)
BinaryResourceMetadata
(
v
(
"@id"
).
str
.
substring
(
s
"$baseUrl/digital/"
.
length
-
1
),
buildDistributorUrl
(
v
(
"@id"
).
str
,
baseUrl
,
distributorHost
,
instantiation
),
if
(
instantiation
==
Thumbnail
)
{
JpegFile
}
else
{
Conversions
.
getMediaFileType
(
v
(
"hasMimeType"
).
str
).
get
},
instantiation
)
case
v
if
isLocalRecord
(
v
)
=>
val
resource
=
MemobaseResource
(
v
(
"type"
).
str
)
throw
new
UnmanageableMediaFileType
(
s
"Media file type for $id unknown"
,
resource
)
...
...
@@ -108,7 +100,6 @@ object BinaryResourceMetadata extends RecordUtils {
throw
new
NoLocalBinary
(
id
,
resource
)
}
)
}
}.
toList
}
src/main/scala/ch/memobase/models/Events.scala
deleted
100644 → 0
View file @
0190b4c8
/*
* Media Converter
* Extracts media files from Fedora repository
* Copyright (C) 2021 Memoriav
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package
ch.memobase.models
/**
* Represents a type of an event produced by Fedora
*/
sealed
trait
Event
/**
* Equals a `create` event type produced by Fedora
*/
case
object
Create
extends
Event
/**
* Equals an `update` event type produced by Fedora
*/
case
object
Update
extends
Event
/**
* Equals a `delete` event type produced by Fedora
*/
case
object
Delete
extends
Event
/**
* If event is unknown
*/
case
class
UnknownEvent
(
eventName
:
String
)
extends
Event
src/main/scala/ch/memobase/models/Exceptions.scala
View file @
4b8ca9ee
...
...
@@ -31,7 +31,3 @@ class NoDigitalObject extends Exception("No digital object found")
class
NoLocalBinary
(
id
:
String
,
val
resource
:
MemobaseResource
)
extends
Exception
(
s
"No reference to local binary in $id found"
)
class
UnknownEventType
(
msg
:
String
)
extends
Exception
(
msg
)
class
NoEventType
(
id
:
String
)
extends
Exception
(
s
"No event type for $id found"
)
src/test/scala/ch/memobase/BinaryResourceMetadataTest.scala
View file @
4b8ca9ee
...
...
@@ -19,7 +19,7 @@
package
ch.memobase
import
ch.memobase.models.
{
BinaryResourceMetadata
,
NoLocalBinary
,
UnknownEventType
,
UnmanageableMediaFileType
}
import
ch.memobase.models.
{
BinaryResourceMetadata
,
NoLocalBinary
,
UnmanageableMediaFileType
}
import
org.scalatest.funsuite.AnyFunSuite
import
scala.io.Source
...
...
@@ -28,10 +28,9 @@ class BinaryResourceMetadataTest extends AnyFunSuite {
val
externalBaseUrl
=
"https://memobase.ch"
private
def
loadMessageWithBinaryResource
(
eventType
:
String
,
mimeType
:
String
,
locator
:
String
)
:
String
=
{
private
def
loadMessageWithBinaryResource
(
mimeType
:
String
,
locator
:
String
)
:
String
=
{
val
file
=
Source
.
fromFile
(
"src/test/resources/incoming_message_with_binary.json"
)
val
result
=
file
.
mkString
.
replaceAll
(
raw
"\{\{eventType\}\}"
,
eventType
)
.
replaceAll
(
raw
"\{\{mimeType\}\}"
,
mimeType
)
.
replaceAll
(
raw
"\{\{locator\}\}"
,
locator
)
file
.
close
()
...
...
@@ -39,13 +38,13 @@ class BinaryResourceMetadataTest extends AnyFunSuite {
}
test
(
"the value of the id field of a KafkaMessage should match the id of the parsed object"
)
{
val
km
=
BinaryResourceMetadata
.
build
(
loadMessageWithBinaryResource
(
"Create"
,
"image/png"
,
"sftp:/BAZ-MEI_77466-1/binary"
),
externalBaseUrl
,
"mb-wf2"
)
val
km
=
BinaryResourceMetadata
.
build
(
loadMessageWithBinaryResource
(
"image/png"
,
"sftp:/BAZ-MEI_77466-1/binary"
),
externalBaseUrl
,
"mb-wf2"
)
assert
(
km
.
head
.
isSuccess
)
}
test
(
"a reference to a non-local binary should throw a NoLocalBinary exception"
)
{
assertThrows
[
NoLocalBinary
]
{
BinaryResourceMetadata
.
build
(
loadMessageWithBinaryResource
(
"Create"
,
"image/jpeg"
,
"https://example.com"
),
externalBaseUrl
,
"mb-wf2"
)
BinaryResourceMetadata
.
build
(
loadMessageWithBinaryResource
(
"image/jpeg"
,
"https://example.com"
),
externalBaseUrl
,
"mb-wf2"
)
.
head
.
get
}
...
...
@@ -53,18 +52,7 @@ class BinaryResourceMetadataTest extends AnyFunSuite {
test
(
"a unmanageable mime type should throw a UnmanageableMediaFileType exception"
)
{
assertThrows
[
UnmanageableMediaFileType
]
{
BinaryResourceMetadata
.
build
(
loadMessageWithBinaryResource
(
"Create"
,
"application/pdf"
,
"sftp:/BAZ-MEI_77466-1/binary"
),
externalBaseUrl
,
"mb-wf2"
)
.
head
.
get
}
}
test
(
"a unknown event type should throw a UnknownEventType exception"
)
{
assertThrows
[
UnknownEventType
]
{
BinaryResourceMetadata
.
build
(
loadMessageWithBinaryResource
(
"Upload"
,
"image/jpeg"
,
BinaryResourceMetadata
.
build
(
loadMessageWithBinaryResource
(
"application/pdf"
,
"sftp:/BAZ-MEI_77466-1/binary"
),
externalBaseUrl
,
"mb-wf2"
)
.
head
.
get
...
...
src/test/scala/ch/memobase/DisseminationCopyHandlerTest.scala
View file @
4b8ca9ee
...
...
@@ -88,16 +88,6 @@ class DisseminationCopyHandlerTest extends AnyFunSuite with BeforeAndAfter {
}
test
(
"calling the deleteAudio function should delete temporary audio file"
)
{
val
f
=
fixture
val
testFile
=
Files
.
createFile
(
Paths
.
get
(
f
.
resPath
,
"test.mp4"
))
val
testSnippetFile
=
Files
.
createFile
(
Paths
.
get
(
f
.
resPath
,
"test-intro.mp3"
))
f
.
fileHandler
.
deleteAudioCopy
(
testFile
.
toString
)
f
.
fileHandler
.
deleteAudioCopy
(
testSnippetFile
.
toString
)
assert
(!
testFile
.
toFile
.
exists
())
assert
(!
testSnippetFile
.
toFile
.
exists
())
}
/**
* ATTENTION: Requires that ffmpeg is properly installed!
*/
...
...
src/test/scala/ch/memobase/RecordProcessorTest.scala
View file @
4b8ca9ee
...
...
@@ -28,20 +28,14 @@ import scala.io.Source
class
RecordProcessorTest
extends
AnyFunSuite
with
MockFactory
{
def
createIncomingMessage
(
eventType
:
Event
,
mimeType
:
MimeType
,
hasBinaryResource
:
Boolean
=
true
)
:
(
String
,
ByteArrayOutputStream
)
=
{
def
createIncomingMessage
(
mimeType
:
MimeType
,
hasBinaryResource
:
Boolean
=
true
)
:
(
String
,
ByteArrayOutputStream
)
=
{
if
(
hasBinaryResource
)
{
val
eT
=
eventType
match
{
case
Create
=>
"Create"
case
Delete
=>
"Delete"
case
Update
=>
"Update"
}
val
mT
=
mimeType
match
{
case
JpegFile
=>
"image/jpeg"
case
Mp3File
=>
"audio/mpeg"
case
VideoMpeg4File
=>
"video/mpeg"
}
replaceTokensInIncomingMessage
(
eT
,
mT
)
replaceTokensInIncomingMessage
(
mT
)
}
else
{
val
is
=
Source
.
fromFile
(
"src/test/resources/incoming_message_without_binary.json"
)
val
incomingMessage
=
is
.
mkString
...
...
@@ -51,11 +45,10 @@ class RecordProcessorTest extends AnyFunSuite with MockFactory {
}
}
private
def
replaceTokensInIncomingMessage
(
eventType
:
String
,
mimeType
:
String
,
locator
:
String
=
"https://memobase.ch/digital/BAZ-MEI_77466-1/binary"
)
=
{
private
def
replaceTokensInIncomingMessage
(
mimeType
:
String
,
locator
:
String
=
"https://memobase.ch/digital/BAZ-MEI_77466-1/binary"
)
=
{
val
is
=
Source
.
fromFile
(
"src/test/resources/incoming_message_with_binary.json"
)
val
incomingMessage
=
is
.
mkString
.
replaceAll
(
raw
"\{\{mimeType\}\}"
,
mimeType
)
.
replaceAll
(
raw
"\{\{eventType\}\}"
,
eventType
)
.
replaceAll
(
raw
"\{\{locator\}\}"
,
locator
)
val
baos
=
copyIncomingMessage
(
incomingMessage
)
is
.
close
...
...
@@ -68,46 +61,4 @@ class RecordProcessorTest extends AnyFunSuite with MockFactory {
baos
.
write
(
incomingMessageAsBytes
,
0
,
incomingMessageAsBytes
.
length
)
baos
}
/* private val fixture = new {
val externalBaseUrl = "https://memobase.ch"
val resourceId = "BAZ-MEI_77466-1"
//val pathToResource = s"$externalBaseUrl/digital/$resourceId/binary"
//val mockDisseminationCopyHandler: DisseminationCopyHandler = mock[DisseminationCopyHandler]
} */
/* test("an object of mimeType image/jpeg and eventType Create should trigger copyImage") {
val f = fixture
val mockDCH = f.mockDisseminationCopyHandler
val mockFCW = f.mockFedoraClientWrapper
val (incomingMessage, baos) = createIncomingMessage(Create, JpegFile)
(mockFCW.fetchBinaryResource _).expects(f.pathToResource).returning(Success(baos))
(mockDCH.createImageCopy _).expects(baos, f.resourceId, JpegFile)
val rP = new RecordProcessor(mockDCH, mockFCW, f.externalBaseUrl)
val cR = new ConsumerRecord[String, String]("_void", 1, 0, "1", incomingMessage)
rP.process(cR)
}
test("an object of mimeType audio/mpeg and eventType Delete should trigger deleteAudio") {
val f = fixture
val mockDCH = f.mockDisseminationCopyHandler
val mockFCW = f.mockFedoraClientWrapper
val (incomingMessage, baos) = createIncomingMessage(Delete, Mp3File)
(mockFCW.fetchBinaryResource _).expects(f.pathToResource).returning(Success(baos))
(mockDCH.deleteAudioCopy _).expects(f.resourceId)
val rP = new RecordProcessor(mockDCH, mockFCW, f.externalBaseUrl)
val cR = new ConsumerRecord[String, String]("_void", 1, 0, "1", incomingMessage)
rP.process(cR)
} */
/* test("a message which does not refer to a binary file should be ignored") {
val f = fixture
val mockDCH = f.mockDisseminationCopyHandler
val (incomingMessage, _) = createIncomingMessage(Delete, Mp3File, hasBinaryResource = false)
val props = new Properties()
props.setProperty("externalBaseUrl", f.externalBaseUrl)
val rP = new RecordProcessor(mockDCH, mockFCW, props)
val cR = new ConsumerRecord[String, String]("_void", 1, 0, "1", incomingMessage)
rP.process(cR)
}*/
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment