Skip to content

GitLab

Commit 208ba180 authored by Jonas Waeber's avatar Jonas Waeber
Browse files

Remove duplicate config files

parent 33e57c29
Show whitespace changes
Inline Side-by-side
deployment/README.md
View file @ 208ba180
......@@ -2,10 +2,7 @@
These files are used to run the migration of the memobase data.
First ensure that the configurations on the SFTP server are on the latest. Use the
script `./configurations/deploy-configurations-prod.sh` to deploy them. This takes a while, because it
sleeps for 10 seconds between each deploy to ensure the SFTP server does not kill the connections.
The configurations for the prod and stage environments are maintained [here](https://gitlab.switch.ch/memoriav/memobase-2020/configurations/import-process).
## Files & Folders
To run these scripts a python installation is required with the package `requests`. The suffix of
the script determines on which infrastructure the request will be executed.
......
deployment/deploy-configurations-prod.sh deleted 100755 → 0
View file @ 33e57c29
#!/usr/bin/env bash
cd prod
for d in **;
do
ssh root@mb-wf2.memobase.unibas.ch "mkdir -p /swissbib_index/mb_sftp/$d/config"
scp $d/mappings/localTransforms.yml root@mb-wf2.memobase.unibas.ch:/swissbib_index/mb_sftp/$d/config/localTransforms.yml;
scp $d/mappings/mapping.yml root@mb-wf2.memobase.unibas.ch:/swissbib_index/mb_sftp/$d/config/mapping.yml;
scp $d/mappings/transform.xslt root@mb-wf2.memobase.unibas.ch:/swissbib_index/mb_sftp/$d/config/transform.xslt;
echo "Copied $d/mappings to $d/config"
sleep 100;
done
ssh root@mb-wf2.memobase.unibas.ch "chown -R mb_sftp:mb_sftp /swissbib_index/mb_sftp/"
\ No newline at end of file
deployment/deploy-specific-configurations-prod.sh deleted 100755 → 0
View file @ 33e57c29
#!/usr/bin/env bash
for d in sag-001;
do
ssh root@mb-wf2.memobase.unibas.ch "mkdir -p /swissbib_index/mb_sftp/$d/config"
scp prod/$d/mappings/localTransforms.yml root@mb-wf2.memobase.unibas.ch:/swissbib_index/mb_sftp/$d/config/localTransforms.yml;
scp prod/$d/mappings/mapping.yml root@mb-wf2.memobase.unibas.ch:/swissbib_index/mb_sftp/$d/config/mapping.yml;
scp prod/$d/mappings/transform.xslt root@mb-wf2.memobase.unibas.ch:/swissbib_index/mb_sftp/$d/config/transform.xslt;
echo "Copied $d/mappings to $d/config"
sleep 10;
done
ssh root@mb-wf2.memobase.unibas.ch "chown -R mb_sftp:mb_sftp /swissbib_index/mb_sftp/"
\ No newline at end of file
deployment/deploy-specific-configurations-stage.sh deleted 100755 → 0
View file @ 33e57c29
#!/usr/bin/env bash
for d in baz-001 cic-001 mfk-001;
do
ssh root@mb-wf2.memobase.unibas.ch "mkdir -p /swissbib_index/mb_sftp/stage/$d/config"
scp stage/$d/mappings/localTransforms.yml root@mb-wf2.memobase.unibas.ch:/swissbib_index/mb_sftp/stage/$d/config/localTransforms.yml;
scp stage/$d/mappings/mapping.yml root@mb-wf2.memobase.unibas.ch:/swissbib_index/mb_sftp/stage/$d/config/mapping.yml;
scp stage/$d/mappings/transform.xslt root@mb-wf2.memobase.unibas.ch:/swissbib_index/mb_sftp/stage/$d/config/transform.xslt;
echo "Copied $d/mappings to $d/config"
sleep 10;
done
ssh root@mb-wf2.memobase.unibas.ch "chown -R mb_sftp:mb_sftp /swissbib_index/mb_sftp/stage"
\ No newline at end of file
deployment/prod/aag-001/mappings/localTransforms.yml deleted 100644 → 0
View file @ 33e57c29
splitEntity:
- type: rico:CorporateBody
property: rico:name
delimiter: ";"
- type: rico:Place
property: rico:name
delimiter: ","
- type: rico:Language
property: rico:name
delimiter: ","
normalizePerson:
creationRelationName: # only tries to extract a value if a DUMMY-VALUE rico:name is property is present in the relation.
pattern: "\\((?<relation>.+)\\)" # " are necessary to ensure pattern is parsed correctly. Pattern needs to be double escaped!
language: NONE
nameOrder: "first-to-last"
singleNameIsLastName: true
nameDelimiter: SPACE
\ No newline at end of file
deployment/prod/aag-001/mappings/mapping.yml deleted 100644 → 0
View file @ 33e57c29
record:
uri: identifierOriginal
identifiers:
original: identifierOriginal
oldMemobase: identifierMain
type: objectType
title: title
titles:
- main: title
- series: SerieTitle
- series: SeriesTitle
- broadcast: BroadcastTitle
scopeAndContent: Context
sameAs: references
abstract: Abstract
descriptiveNote: Remarks
source: source
relation: relatedMaterial
isSponsoredByMemoriav: true
rights:
holder: rightsHolder
languages:
- content: language
- caption: captioningFormat
subject:
- prefLabel: Keywords
genre:
- prefLabel: genre
placeOfCapture:
- name: RecordingLocation
relatedPlaces:
- name: location
creationDate: createdDate
issuedDate: issuedDate
temporal: PeriodOfTimeDate
creators:
- person:
name: creatorPerson.name
relationName: creatorPerson.role
- corporateBody:
name: creatorCorporateBody.name
relationName: creatorCorporateBody.role
- agent:
name: creatorAgent.name
relationName: creatorAgent.role
contributors:
- person:
name: contributorPerson.name
relationName: contributorPerson.role
- corporateBody:
name: contributorCorporateBody.name
relationName: contributorCorporateBody.role
- agent:
name: contributorAgent.name
relationName: contributorAgent.role
producers:
- person:
name: producerPersonName
- corporateBody:
name: producerCorporateBodyName
relatedAgents:
- person:
name: FeaturedIndividual
- corporateBody:
name: FeaturedOrganisation
publishedBy:
- person:
name: publisherPerson.name
- corporateBody:
name: publisherCorporateBody.name
conditionsOfUse: rightsUsage
physical:
carrierType: medium
duration: duration
physicalCharacteristics:
- prefix:
value: "Startposition: "
field: start
- prefix:
value: "Seitenverhältnis: "
field: videoFormatRatio
- prefix:
value: "TV Norm: "
field: videoFormatTVSystem
- prefix:
value: "Bemerkung: "
field: videoFormatRemarks
- prefix:
value: "Verfahren: "
field: imageFormatMode
- prefix:
value: "Bemerkung: "
field: imageFormatRemarks
- prefix:
value: "Abspielgeschwindigkeit: "
field: audioFormatPlaybackSpeed
- prefix:
value: "Bemerkung: "
field: audioFormatRemarks
- prefix:
value: "Weite: "
field: width
- prefix:
value: "Höhe: "
field: height
- prefix:
value: "Tonaufnahmeverfahren: "
field: audioTrackConfiguration
colour:
- videoFormatColor
- videoFormatColorMode
- imageFormatColorMode
- imageFormatColor
identifiers:
callNumber: identifierCallNumber
rights:
access: accessPhysical
usage:
name:
const: "Copyright Not Evaluated (CNE)"
sameAs:
const: "http://rightsstatements.org/vocab/CNE/1.0/"
digital:
locator: essenceLocator
rights:
access: accessDigital
usage:
name: rightsStatementText
sameAs: rightsStatementURL
\ No newline at end of file
deployment/prod/aag-001/mappings/transform.xslt deleted 100644 → 0
View file @ 33e57c29
<?xml version="1.0" encoding="UTF-8"?>
<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
xmlns:fn="http://www.w3.org/2005/xpath-functions"
version="2.0"
xmlns:ns2="http://purl.org/dc/elements/1.1/"
xmlns:foxml="info:fedora/fedora-system:def/foxml#"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:audit="info:fedora/fedora-system:def/audit#"
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#"
xmlns:fedora="info:fedora/fedora-system:def/relations-external#"
xmlns:fedora-model="info:fedora/fedora-system:def/model#"
xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:xslt="http://www.w3.org/1999/XSL/Transform"
xmlns:core="urn:ebu:metadata-schema:ebuCore_2012"
xpath-default-namespace="urn:ebu:metadata-schema:ebuCore_2012">
<xsl:output
indent="yes"
method="xml"
/>
<!--<xsl:template match="/digitalObject/datastream/datastreamVersion/xmlContent/ebuCoreMain/coreMetadata">
<xsl:copy-of select="."/>
</xsl:template>-->
<xsl:template match="@*|node()">
<xsl:apply-templates select="@*|node()"/>
</xsl:template>
<xsl:template match="core:coreMetadata">
<xsl:element name="record">
<!--<xsl:apply-templates select="test"/>
<xsl:apply-templates select="title, ns2:title, alternativeTitle, subject, description, format/essenceLocator, format/medium, type/*, references"/>-->
<xsl:apply-templates />
</xsl:element>
</xsl:template>
<!-- content of child node is copied to parent node -->
<xsl:template match="title | format/essenceLocator | format/duration | format/start | language | references | coverage/spatial/location">
<xsl:element name="{local-name()}">
<xsl:value-of select="child::*"/>
</xsl:element>
</xsl:template>
<!-- content of typeLabel is transformed to an element name and content of child node is copied to it-->
<xsl:template match="description[@typeLabel] | alternativeTitle[@typeLabel] | subject[@typeLabel] | relation[@typeLabel]">
<xsl:variable name="typeLabel" select="@typeLabel"/>
<xsl:element name="{$typeLabel}">
<xsl:value-of select="child::*"/>
</xsl:element>
</xsl:template>
<!-- content of typeLabel is copied as content of the node -->
<xsl:template match="format/medium | format/dataFormat/captioningFormat | type/genre">
<xsl:element name="{local-name()}">
<xsl:value-of select="@typeLabel | @language"/>
</xsl:element>
</xsl:template>
<xsl:template match="type/objectType">
<xsl:element name="objectType">
<xsl:choose>
<xsl:when test="matches(@typeLabel,'film','i')">
<xsl:text>Film</xsl:text>
</xsl:when>
<xsl:when test="matches(@typeLabel,'photograph|foto','i')">
<xsl:text>Foto</xsl:text>
</xsl:when>
<xsl:when test="matches(@typeLabel,'radio','i')">
<xsl:text>Radio</xsl:text>
</xsl:when>
<xsl:when test="matches(@typeLabel,'television|tv','i')">
<xsl:text>TV</xsl:text>
</xsl:when>
<xsl:when test="matches(@typeLabel,'ton|sound','i')">
<xsl:text>Ton</xsl:text>
</xsl:when>
<xsl:when test="matches(@typeLabel,'video','i')">
<xsl:text>Video</xsl:text>
</xsl:when>
<xsl:when test="matches(@typeLabel,'tbs','i')">
<xsl:text>Tonbildschau</xsl:text>
</xsl:when>
</xsl:choose>
</xsl:element>
</xsl:template>
<xsl:template match="identifier">
<xsl:variable name="typeLabel" select="@typeLabel"/>
<xsl:element name="identifier{$typeLabel}">
<xsl:value-of select="child::ns2:identifier"/>
</xsl:element>
</xsl:template>
<xsl:template match="format/videoFormat | format/imageFormat | format/audioFormat">
<xsl:variable name="format" select="local-name()"/>
<xsl:for-each select="technicalAttributeString">
<xsl:variable name="typeLabel" select="@typeLabel"/>
<xsl:element name="{$format}{$typeLabel}">
<xsl:value-of select="."/>
</xsl:element>
</xsl:for-each>
<xsl:for-each select="width | height">
<xsl:variable name="local" select="local-name()"/>
<xsl:variable name="unit" select="@unit"/>
<xsl:element name="{$local}">
<xsl:value-of select="concat(., ' ', $unit)"/>
</xsl:element>
</xsl:for-each>
<xsl:for-each select="audioTrackConfiguration">
<xsl:element name="audioTrackConfiguration">
<xsl:value-of select="@typeLabel"/>
</xsl:element>
</xsl:for-each>
</xsl:template>
<xsl:template match="contributor | creator | publisher">
<xsl:variable name="type" select="local-name()"/>
<xsl:for-each select=".">
<xsl:variable name="role" select="child::role/@typeLabel[. != '']"/>
<xsl:choose>
<xsl:when test="$role = 'ResponsibleInstitution'"/>
<xsl:when test="$role = 'Producer' or $role = 'Production' or $role = 'Produktion'">
<xsl:choose>
<xsl:when test="child::organisationDetails">
<xsl:element name="producerCorporateBodyName">
<xsl:value-of select="descendant::organisationName"/>
</xsl:element>
</xsl:when>
<xsl:when test="child::contactDetails">
<xsl:element name="producerPersonName">
<xsl:value-of select="descendant::name"/>
</xsl:element>
</xsl:when>
</xsl:choose>
</xsl:when>
<!-- To be used for all SRF RecordSets where a Redaktion is recorded as Person in the data -->
<!--<xsl:when test="$role = 'Redaktion'">
<xsl:element name="{$type}CorporateBody">
<xsl:element name="name">
<xsl:value-of select="descendant::name"/>
</xsl:element>
<xsl:if test="$role">
<xsl:element name="role">
<xsl:value-of select="$role"/>
</xsl:element>
</xsl:if>
</xsl:element>
</xsl:when>-->
<!-- To be used for all Tanzarchiv RecordSets where a Kompanie is recorded as Person in the data -->
<!--<xsl:when test="$role = 'Kompanie / Compagnie'">
<xsl:element name="{$type}CorporateBody">
<xsl:element name="name">
<xsl:value-of select="descendant::name"/>
</xsl:element>
<xsl:if test="$role">
<xsl:element name="role">
<xsl:value-of select="$role"/>
</xsl:element>
</xsl:if>
</xsl:element>
</xsl:when>-->
<xsl:when test="child::organisationDetails">
<xsl:element name="{$type}CorporateBody">
<xsl:element name="name">
<xsl:value-of select="descendant::organisationName"/>
</xsl:element>
<xsl:if test="$role">
<xsl:element name="role">
<xsl:value-of select="$role"/>
</xsl:element>
</xsl:if>
</xsl:element>
</xsl:when>
<xsl:when test="child::contactDetails">
<xsl:choose>
<xsl:when test="matches(descendant::name,'^unbekannt')">
<xsl:element name="{$type}Agent">
<xsl:element name="name">
<xsl:value-of select="descendant::name"/>
</xsl:element>
<xsl:if test="$role">
<xsl:element name="role">
<xsl:value-of select="$role"/>
</xsl:element>
</xsl:if>
</xsl:element>
</xsl:when>
<xsl:otherwise>
<xsl:element name="{$type}Person">
<xsl:element name="name">
<xsl:value-of select="descendant::name"/>
</xsl:element>
<xsl:if test="$role">
<xsl:element name="role">
<xsl:value-of select="$role"/>
</xsl:element>
</xsl:if>
</xsl:element>
</xsl:otherwise>
</xsl:choose>
</xsl:when>
</xsl:choose>
</xsl:for-each>
</xsl:template>
<xsl:template match="rights[@typeLabel='Access']/ns2:rights">
<xsl:if test="matches(.,'onsite')">
<xsl:element name="accessPhysical">
<xsl:text>onsite</xsl:text>
</xsl:element>
</xsl:if>
<xsl:if test="matches(.,'noonsite')">
<xsl:element name="accessPhysical">
<xsl:text>noonsite</xsl:text>
</xsl:element>
</xsl:if>
<xsl:if test="matches(.,'public')">
<xsl:element name="accessDigital">
<xsl:text>public</xsl:text>
</xsl:element>
<xsl:element name="rightsStatementText">
<xsl:text>Copyright Not Evaluated (CNE)</xsl:text>
</xsl:element>
<xsl:element name="rightsStatementURL">
<xsl:text>http://rightsstatements.org/vocab/CNE/1.0/</xsl:text>
</xsl:element>
</xsl:if>
<xsl:if test="matches(.,'private')">
<xsl:element name="accessDigital">
<xsl:text>private</xsl:text>
</xsl:element>
<xsl:element name="rightsStatementText">
<xsl:text>Copyright Not Evaluated (CNE)</xsl:text>
</xsl:element>
<xsl:element name="rightsStatementURL">
<xsl:text>http://rightsstatements.org/vocab/CNE/1.0/</xsl:text>
</xsl:element>
</xsl:if>
<xsl:if test="matches(.,'faro')">
<xsl:element name="accessDigital">
<xsl:text>faro</xsl:text>
</xsl:element>
<xsl:element name="rightsStatementText">
<xsl:text>Copyright Not Evaluated (CNE)</xsl:text>
</xsl:element>
<xsl:element name="rightsStatementURL">
<xsl:text>http://rightsstatements.org/vocab/CNE/1.0/</xsl:text>
</xsl:element>
</xsl:if>
</xsl:template>
<xsl:template match="rights[@typeLabel='Holder']/rightsHolder/contactDetails/name | rights[@typeLabel='Holder']/ns2:rights">
<xsl:element name="rightsHolder">
<xsl:value-of select="."/>
</xsl:element>
</xsl:template>
<xsl:template match="rights[@typeLabel='Usage']/ns2:rights">
<xsl:element name="rightsUsage">
<xsl:value-of select="."/>
</xsl:element>
</xsl:template>
<xsl:template match="core:coreMetadata/ns2:source">
<xsl:element name="source">
<xsl:value-of select="."/>
</xsl:element>
</xsl:template>
<xsl:template match="date/created | date/issued | temporal/PeriodOfTime">
<xsl:variable name="date" select="local-name()"/>
<xsl:for-each select=".">
<xsl:element name="{$date}Date">
<xsl:choose>
<xsl:when test="descendant::periodName">
<xsl:value-of select="descendant::periodName"/>
</xsl:when>
<xsl:otherwise>
<xsl:call-template name="dates"/>
</xsl:otherwise>
</xsl:choose>
</xsl:element>
</xsl:for-each>
</xsl:template>
<xsl:template name="dates">
<xsl:variable name="startDate" select="@startDate[. != '']"/>
<xsl:variable name="startYear" select="@startYear[. != '']"/>
<xsl:variable name="startTime" select="@startTime[. != '']"/>
<xsl:variable name="endDate" select="@endDate[. != '']"/>
<xsl:variable name="endYear" select="@endYear[. != '']"/>
<xsl:variable name="period" select="@period[. != '']"/>
<xsl:choose>
<!-- Combinations which are not present in created/issued/temporal and are not processed:
* endYear and startDate
* period and endDate and startYear
* period and endYear and startDate
* startTime in another combination but with startDate
* period and endDate only
-->
<xsl:when test="$startDate and not($endDate | $endYear | $period | $startTime)">
<xsl:value-of select="$startDate"/>
</xsl:when>
<xsl:when test="$startYear and not($endDate | $endYear | $period | $startTime)">
<xsl:value-of select="$startYear"/>
</xsl:when>
<xsl:when test="$startDate and $endDate and not($period | $startTime)">
<xsl:value-of select="concat($startDate, '/', $endDate)"/>
</xsl:when>
<xsl:when test="$startYear and $endDate and not($period | $startTime)">
<xsl:value-of select="concat($startYear, '/', $endDate)"/>
</xsl:when>
<xsl:when test="$startYear and $endYear and not($period | $startTime)">
<xsl:value-of select="concat($startYear, '/', $endYear)"/>
</xsl:when>
<xsl:when test="$startTime">
<xsl:value-of select="concat($startDate, 'T', $startTime)"/>
</xsl:when>
<xsl:when test="$endDate and not ($startDate | $startYear | $period)">
<xsl:value-of select="concat('?/', $endDate)"/>
</xsl:when>
<xsl:when test="$endYear and not ($startDate | $startYear | $period)">
<xsl:value-of select="concat('?/', $endYear)"/>
</xsl:when>
<xsl:when test="$period and not($endDate | $endYear | $startDate | $startYear)">
<xsl:value-of select="$period"/>
</xsl:when>
<xsl:when test="$period and $startDate and not ($endDate | $endYear)">
<xsl:value-of select="concat($period, ' ', $startDate)"/>
</xsl:when>
<xsl:when test="$period and $startYear and not ($endDate | $endYear)">
<xsl:value-of select="concat($period, ' ', $startYear)"/>
</xsl:when>
<xsl:when test="$period and $startDate and $endDate">
<xsl:value-of select="concat($period, ' ', $startDate, '/', $endDate)"/>
</xsl:when>
<xsl:when test="$period and $startYear and $endYear">
<xsl:choose>
<xsl:when test="matches(@period,concat(@startYear, '-', @endYear))">
<xsl:value-of select="concat($startYear, '/', $endYear)"/>
</xsl:when>
<xsl:otherwise>
<xsl:value-of select="concat($period, ' ', $startYear, '/', $endYear)"/>
</xsl:otherwise>
</xsl:choose>
</xsl:when>
</xsl:choose>
</xsl:template>
<!-- field to delete -->
<xsl:template match="isMemberOf "/>
</xsl:stylesheet>
\ No newline at end of file
deployment/prod/abe-001/mappings/localTransforms.yml deleted 100644 → 0
View file @ 33e57c29
splitEntity:
- type: skos:Concept
property: skos:prefLabel
delimiter: ","
- type: rico:CorporateBody
property: rico:name
delimiter: ";"
- type: rico:Place
property: rico:name
delimiter: "/"
- type: rico:Place
property: rico:name
delimiter: ","
- type: rico:Language
property: rico:name
delimiter: ","
normalizePerson:
creationRelationName: # only tries to extract a value if a DUMMY-VALUE rico:name is property is present in the relation.
pattern: "\\((?<relation>.+)\\)" # " are necessary to ensure pattern is parsed correctly. Pattern needs to be double escaped!
language: NONE
nameOrder: "first-to-last"
singleNameIsLastName: true
nameDelimiter: SPACE
\ No newline at end of file
deployment/prod/abe-001/mappings/mapping.yml deleted 100644 → 0
View file @ 33e57c29
record:
uri: identifierOriginal
identifiers: