Add "unsafe fixes" runtime option

In this case it fixes occurences of invalid multi-value separators.
DSpace uses "||" to separate multiple values in one field, but our
editors sometimes give us files with mistakes like "|". We can fix
these to be correct multi-value separators if we are sure that the
metadata is not actually using "|" for some legitimate purpose.

csv_metadata_quality/app.py

@@ -9,6 +9,7 @@ def parse_args(argv):
     parser = argparse.ArgumentParser(description='Metadata quality checker and fixer.')
     parser.add_argument('--input-file', '-i', help='Path to input file. Can be UTF-8 CSV or Excel XLSX.', required=True, type=argparse.FileType('r', encoding='UTF-8'))
     parser.add_argument('--output-file', '-o', help='Path to output file (always CSV).', required=True, type=argparse.FileType('w', encoding='UTF-8'))
+    parser.add_argument('--unsafe-fixes', '-u', help='Perform unsafe fixes.', action='store_true')
     args = parser.parse_args()
 
     return args
@@ -28,6 +29,12 @@ def main(argv):
         # Run invalid multi-value separator check on all columns
         df[column] = df[column].apply(check.separators)
 
+        # Run invalid multi-value separator fix on all columns
+        if args.unsafe_fixes:
+            df[column] = df[column].apply(fix.separators)
+            # Run whitespace fix again after fixing invalid separators
+            df[column] = df[column].apply(fix.whitespace)
+
         # check if column is an issn column like dc.identifier.issn
         match = re.match(r'^.*?issn.*$', column)
         if match is not None:

csv_metadata_quality/fix.py

@@ -35,3 +35,33 @@ def whitespace(field):
     new_field = '||'.join(values)
 
     return new_field
+
+
+def separators(field):
+    """Fix for invalid multi-value separators (ie "|")."""
+
+    # Skip fields with missing values
+    if pd.isna(field):
+        return
+
+    # Initialize an empty list to hold the cleaned values
+    values = list()
+
+    # Try to split multi-value field on "||" separator
+    for value in field.split('||'):
+        # After splitting, see if there are any remaining "|" characters
+        pattern = re.compile(r'\|')
+        match = re.findall(pattern, value)
+
+        if len(match) > 0:
+            print(f'Fixing invalid multi-value separator: {value}')
+
+            value = re.sub(pattern, '||', value)
+
+        # Save cleaned value
+        values.append(value)
+
+    # Create a new field consisting of all values joined with "||"
+    new_field = '||'.join(values)
+
+    return new_field

data/test.csv

@@ -4,4 +4,4 @@ Stella|| Stella ||Stella Orth||Stella ,1984-11-27,2321-2302,99921-58-10-7
 Sophia,2019-06-15,,
 Test,2019-06-150,,
 "Doe, J.",2019-06-15||2019-01-10,,
-Someone,,,
+Someone,,0378-5955|0378-5955,

tests/test_fix.py

@@ -23,3 +23,11 @@ def test_fix_excessive_whitespace():
     value = 'Alan  Orth'
 
     assert fix.whitespace(value) == 'Alan Orth'
+
+
+def test_fix_invalid_separators():
+    '''Test fixing invalid multi-value separators.'''
+
+    value = 'Alan|Orth'
+
+    assert fix.separators(value) == 'Alan||Orth'