1
0
mirror of https://github.com/ilri/csv-metadata-quality.git synced 2024-07-15 09:11:19 +02:00

Add "unsafe fixes" runtime option

In this case it fixes occurences of invalid multi-value separators.
DSpace uses "||" to separate multiple values in one field, but our
editors sometimes give us files with mistakes like "|". We can fix
these to be correct multi-value separators if we are sure that the
metadata is not actually using "|" for some legitimate purpose.
This commit is contained in:
Alan Orth 2019-07-28 22:53:39 +03:00
parent a93b5b31c5
commit 40e77db713
Signed by: alanorth
GPG Key ID: 0FB860CC9C45B1B9
4 changed files with 46 additions and 1 deletions

View File

@ -9,6 +9,7 @@ def parse_args(argv):
parser = argparse.ArgumentParser(description='Metadata quality checker and fixer.')
parser.add_argument('--input-file', '-i', help='Path to input file. Can be UTF-8 CSV or Excel XLSX.', required=True, type=argparse.FileType('r', encoding='UTF-8'))
parser.add_argument('--output-file', '-o', help='Path to output file (always CSV).', required=True, type=argparse.FileType('w', encoding='UTF-8'))
parser.add_argument('--unsafe-fixes', '-u', help='Perform unsafe fixes.', action='store_true')
args = parser.parse_args()
return args
@ -28,6 +29,12 @@ def main(argv):
# Run invalid multi-value separator check on all columns
df[column] = df[column].apply(check.separators)
# Run invalid multi-value separator fix on all columns
if args.unsafe_fixes:
df[column] = df[column].apply(fix.separators)
# Run whitespace fix again after fixing invalid separators
df[column] = df[column].apply(fix.whitespace)
# check if column is an issn column like dc.identifier.issn
match = re.match(r'^.*?issn.*$', column)
if match is not None:

View File

@ -35,3 +35,33 @@ def whitespace(field):
new_field = '||'.join(values)
return new_field
def separators(field):
"""Fix for invalid multi-value separators (ie "|")."""
# Skip fields with missing values
if pd.isna(field):
return
# Initialize an empty list to hold the cleaned values
values = list()
# Try to split multi-value field on "||" separator
for value in field.split('||'):
# After splitting, see if there are any remaining "|" characters
pattern = re.compile(r'\|')
match = re.findall(pattern, value)
if len(match) > 0:
print(f'Fixing invalid multi-value separator: {value}')
value = re.sub(pattern, '||', value)
# Save cleaned value
values.append(value)
# Create a new field consisting of all values joined with "||"
new_field = '||'.join(values)
return new_field

View File

@ -4,4 +4,4 @@ Stella|| Stella ||Stella Orth||Stella ,1984-11-27,2321-2302,99921-58-10-7
Sophia,2019-06-15,,
Test,2019-06-150,,
"Doe, J.",2019-06-15||2019-01-10,,
Someone,,,
Someone,,0378-5955|0378-5955,

1 dc.contributor.author birthdate dc.identifier.issn dc.identifier.isbn
4 Sophia 2019-06-15
5 Test 2019-06-150
6 Doe, J. 2019-06-15||2019-01-10
7 Someone 0378-5955|0378-5955

View File

@ -23,3 +23,11 @@ def test_fix_excessive_whitespace():
value = 'Alan Orth'
assert fix.whitespace(value) == 'Alan Orth'
def test_fix_invalid_separators():
'''Test fixing invalid multi-value separators.'''
value = 'Alan|Orth'
assert fix.separators(value) == 'Alan||Orth'