Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
memoriav
Memobase 2020
services
postprocessing
Media Metadata Indexer
Commits
f93d74f8
Unverified
Commit
f93d74f8
authored
Nov 27, 2020
by
Sebastian Schüpbach
Browse files
write report in every case
parent
045408f3
Pipeline
#17932
passed with stages
in 1 minute and 52 seconds
Changes
2
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
mediametadatatodb_app/resources/MediametadataToDB.py
View file @
f93d74f8
...
...
@@ -79,30 +79,22 @@ def _connect_to_mariadb(retries=0):
def
_try_fetch_from_json_object
(
record_json_data
,
record_values_for_db
,
fetch_from_obj_fun
,
access_status
):
if
'locator'
in
record_json_data
:
values
=
fetch_from_obj_fun
(
record_json_data
,
access_status
)
record_values_for_db
.
append
(
values
)
if
'locator'
in
record_json_data
and
'@id'
in
record_json_data
:
return
fetch_from_obj_fun
(
record_json_data
,
access_status
)
elif
'@id'
in
record_json_data
:
logging
.
info
(
'Record '
+
record_json_data
[
'@id'
]
+
' does not have a locator-property.'
)
else
:
logging
.
warning
(
'Record without @id-property detected!'
)
return
None
def
_get_values_from_thumbnail_object
(
msg
,
_access_status
):
return_values
=
{
'mimetype'
:
'image/jp2'
,
'type'
:
'image'
,
'access'
:
'public'
,
'proto'
:
'file'
}
if
'@id'
in
msg
:
return_values
[
'sig'
]
=
'{}-poster'
.
format
(
msg
[
'@id'
].
split
(
'/'
)[
-
2
])
if
'locator'
in
msg
:
return_values
[
'uri'
]
=
'file:///data/{}-poster.jp2'
.
format
(
msg
[
'@id'
].
split
(
'/'
)[
-
2
])
return_values
=
{
'mimetype'
:
'image/jp2'
,
'type'
:
'image'
,
'access'
:
'public'
,
'proto'
:
'file'
,
'sig'
:
'{}-poster'
.
format
(
msg
[
'@id'
].
split
(
'/'
)[
-
2
]),
'uri'
:
'file:///data/{}-poster.jp2'
.
format
(
msg
[
'@id'
].
split
(
'/'
)[
-
2
])}
if
'height'
in
msg
:
height
=
msg
[
'height'
]
return_values
[
'height'
]
=
height
...
...
@@ -113,12 +105,11 @@ def _get_values_from_thumbnail_object(msg, _access_status):
def
_get_values_from_digital_object
(
msg
,
access_status
):
if
'isDistributedOn'
not
in
msg
:
logging
.
warning
(
"No isDistributedOn property found in object"
)
return
None
file_extension
=
''
return_values
=
{
'access'
:
access_status
}
if
'@id'
in
msg
:
return_values
[
'sig'
]
=
msg
[
'@id'
].
split
(
'/'
)[
-
1
]
return_values
=
{
'access'
:
access_status
,
'sig'
:
msg
[
'@id'
].
split
(
'/'
)[
-
1
]}
if
'height'
in
msg
:
height
=
msg
[
'height'
]
return_values
[
'height'
]
=
height
...
...
@@ -128,8 +119,7 @@ def _get_values_from_digital_object(msg, access_status):
if
'duration'
in
msg
:
duration
=
msg
[
'duration'
]
return_values
[
'duration'
]
=
duration
if
'isDistributedOn'
in
msg
:
return_values
[
'type'
]
=
msg
[
'isDistributedOn'
]
return_values
[
'type'
]
=
msg
[
'isDistributedOn'
]
if
'hasMimeType'
in
msg
:
if
return_values
[
'type'
]
==
'image'
:
mimetype
=
'image/jp2'
...
...
@@ -224,6 +214,7 @@ def _has_audio_snippet(record):
record
[
'uri'
].
startswith
(
'file://'
)
# TODO: Eventually remove
def
_has_http_locator
(
digital_object
):
'locator'
in
digital_object
and
digital_object
[
'locator'
].
startswith
(
'http'
)
...
...
@@ -291,17 +282,38 @@ class MediametadataToDB:
access_status
=
_get_access_status
(
records_json_data
,
record_id
)
if
access_status
==
'public'
or
access_status
==
'closed'
:
for
recordJsonData
in
records_json_data
:
enrichable
=
False
if
'type'
in
recordJsonData
and
\
recordJsonData
[
'type'
]
==
'digitalObject'
and
\
_has_http_locator
(
recordJsonData
):
_try_fetch_from_json_object
(
recordJsonData
,
record_values_for_db
,
_get_values_from_digital_object
,
access_status
)
enrichable
=
True
enriched_data
=
\
_try_fetch_from_json_object
(
recordJsonData
,
_get_values_from_digital_object
,
access_status
)
if
not
enriched_data
:
reporter
.
send_message
(
record_id
,
"FATAL"
,
"Could not process digitalObject"
)
else
:
record_values_for_db
.
append
(
enriched_data
)
if
'type'
in
recordJsonData
and
\
recordJsonData
[
'type'
]
==
'thumbnail'
:
_try_fetch_from_json_object
(
recordJsonData
,
record_values_for_db
,
_get_values_from_thumbnail_object
,
access_status
)
enrichable
=
True
enriched_data
=
\
_try_fetch_from_json_object
(
recordJsonData
,
_get_values_from_thumbnail_object
,
access_status
)
if
not
enriched_data
:
reporter
.
send_message
(
record_id
,
"FATAL"
,
"Could not process thumbnail object"
)
else
:
record_values_for_db
.
append
(
enriched_data
)
if
not
enrichable
:
reporter
.
send_message
(
record_id
,
"IGNORE"
,
"Resource has no digitalObject or thumbnail"
)
else
:
logging
.
info
(
f
'Ignoring record
{
record_id
}
since'
+
f
' access of digitalObject is
{
access_status
}
'
)
...
...
tests/unit/test_MediametadataToDB.py
View file @
f93d74f8
...
...
@@ -130,10 +130,11 @@ class Test(TestCase):
Test
.
_load_file_and_get_res
(
'SRF-UMusik-BE_MG_DUR_B62276.json'
,
lambda
x
:
x
):
if
'type'
in
recordJsonData
and
\
recordJsonData
[
'type'
]
==
'digitalObject'
:
MediametadataToDB
.
_try_fetch_from_json_object
(
recordJsonData
,
record_values_for_db
,
MediametadataToDB
.
_get_values_from_digital_object
,
'public'
)
record_values_for_db
.
append
(
MediametadataToDB
.
_try_fetch_from_json_object
(
recordJsonData
,
MediametadataToDB
.
_get_values_from_digital_object
,
'public'
))
self
.
assertFalse
([
MediametadataToDB
.
_has_audio_snippet
(
record
)
for
record
in
record_values_for_db
][
0
])
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment