Version 0.2.1

Add a check for soft hyphens (U+00AD). In one sample CSV I have a
normal hyphen followed by a soft hyphen in an ISBN. This causes the
ISBN validation to fail.

CHANGELOG.md

@@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.2.1] - 2019-08-11
+### Added
+- Check for uncommon filename extensions 
+- Replacement of unneccessary Unicode characters like soft hyphens (U+00AD)
+
 ## [0.2.0] - 2019-08-09
 ### Added
 - Handle Ctrl-C interrupt gracefully

csv_metadata_quality/app.py

@@ -86,6 +86,10 @@ def run(argv):
         if match is not None:
             df[column] = df[column].apply(check.date)
 
+        # Check: filename extension
+        if column == 'filename':
+            df[column] = df[column].apply(check.filename_extension)
+
     # Write
     df.to_csv(args.output_file, index=False)
 

csv_metadata_quality/check.py

@@ -253,3 +253,48 @@ def agrovoc(field, field_name):
                     print(f'Invalid AGROVOC ({field_name}): {value}')
 
     return field
+
+
+def filename_extension(field):
+    """Check filename extension.
+
+    CSVs with a 'filename' column are likely meant as input for the SAFBuilder
+    tool, which creates a Simple Archive Format bundle for importing metadata
+    with accompanying PDFs or other files into DSpace.
+
+    This check warns if a filename has an uncommon extension (that is, other
+    than .pdf, .xls(x), .doc(x), ppt(x), case insensitive).
+    """
+
+    import re
+
+    # Skip fields with missing values
+    if pd.isna(field):
+        return
+
+    # Try to split multi-value field on "||" separator
+    values = field.split('||')
+
+    # List of common filename extentions
+    common_filename_extensions = ['.pdf', '.doc', '.docx', '.ppt', '.pptx', '.xls', '.xlsx']
+
+    # Iterate over all values
+    for value in values:
+        # Assume filename extension does not match
+        filename_extension_match = False
+
+        for filename_extension in common_filename_extensions:
+            # Check for extension at the end of the filename
+            pattern = re.escape(filename_extension) + r'$'
+            match = re.search(pattern, value, re.IGNORECASE)
+
+            if match is not None:
+                # Register the match and stop checking for this filename
+                filename_extension_match = True
+
+                break
+
+        if filename_extension_match is False:
+            print(f'Filename with uncommon extension: {value}')
+
+    return field

csv_metadata_quality/fix.py

@@ -68,14 +68,17 @@ def separators(field):
 
 
 def unnecessary_unicode(field):
-    """Remove unnecessary Unicode characters.
+    """Remove and replace unnecessary Unicode characters.
 
     Removes unnecessary Unicode characters like:
         - Zero-width space (U+200B)
         - Replacement character (U+FFFD)
         - No-break space (U+00A0)
 
-    Return string with characters removed.
+    Replaces unnecessary Unicode characters like:
+        - Soft hyphen (U+00AD) → hyphen
+
+    Return string with characters removed or replaced.
     """
 
     # Skip fields with missing values
@@ -106,6 +109,14 @@ def unnecessary_unicode(field):
         print(f'Removing unnecessary Unicode (U+00A0): {field}')
         field = re.sub(pattern, '', field)
 
+    # Check for soft hyphens (U+00AD), sometimes preceeded with a normal hyphen
+    pattern = re.compile(r'\u002D*?\u00AD')
+    match = re.findall(pattern, field)
+
+    if match:
+        print(f'Replacing unnecessary Unicode (U+00AD): {field}')
+        field = re.sub(pattern, '-', field)
+
     return field
 
 

csv_metadata_quality/version.py

@@ -1 +1 @@
-VERSION = '0.2.0'
+VERSION = '0.2.1'

data/test.csv

@@ -1,23 +1,25 @@
-dc.contributor.author,birthdate,dc.identifier.issn,dc.identifier.isbn,dc.language.iso,dc.subject,cg.coverage.country
- Leading space,2019-07-29,,,,,
-Trailing space ,2019-07-29,,,,,
-Excessive  space,2019-07-29,,,,,
-Miscellaenous ||whitespace | issues ,2019-07-29,,,,,
-Duplicate||Duplicate,2019-07-29,,,,,
-Invalid ISSN,2019-07-29,2321-2302,,,,
-Invalid ISBN,2019-07-29,,978-0-306-40615-6,,,
-Multiple valid ISSNs,2019-07-29,0378-5955||0024-9319,,,,
-Multiple valid ISBNs,2019-07-29,,99921-58-10-7||978-0-306-40615-7,,,
-Invalid date,2019-07-260,,,,,
-Multiple dates,2019-07-26||2019-01-10,,,,,
-Invalid multi-value separator,2019-07-29,0378-5955|0024-9319,,,,
-Unnecessary Unicode​,2019-07-29,,,,,
-Suspicious character||foreˆt,2019-07-29,,,,,
-Invalid ISO 639-2 language,2019-07-29,,,jp,,
-Invalid ISO 639-3 language,2019-07-29,,,chi,,
-Invalid language,2019-07-29,,,Span,,
-Invalid AGROVOC subject,2019-07-29,,,,FOREST,
+dc.contributor.author,birthdate,dc.identifier.issn,dc.identifier.isbn,dc.language.iso,dc.subject,cg.coverage.country,filename
+ Leading space,2019-07-29,,,,,,
+Trailing space ,2019-07-29,,,,,,
+Excessive  space,2019-07-29,,,,,,
+Miscellaenous ||whitespace | issues ,2019-07-29,,,,,,
+Duplicate||Duplicate,2019-07-29,,,,,,
+Invalid ISSN,2019-07-29,2321-2302,,,,,
+Invalid ISBN,2019-07-29,,978-0-306-40615-6,,,,
+Multiple valid ISSNs,2019-07-29,0378-5955||0024-9319,,,,,
+Multiple valid ISBNs,2019-07-29,,99921-58-10-7||978-0-306-40615-7,,,,
+Invalid date,2019-07-260,,,,,,
+Multiple dates,2019-07-26||2019-01-10,,,,,,
+Invalid multi-value separator,2019-07-29,0378-5955|0024-9319,,,,,
+Unnecessary Unicode​,2019-07-29,,,,,,
+Suspicious character||foreˆt,2019-07-29,,,,,,
+Invalid ISO 639-2 language,2019-07-29,,,jp,,,
+Invalid ISO 639-3 language,2019-07-29,,,chi,,,
+Invalid language,2019-07-29,,,Span,,,
+Invalid AGROVOC subject,2019-07-29,,,,FOREST,,
 Newline (LF),2019-07-30,,,,"TANZA
-NIA",
-Missing date,,,,,,
-Invalid country,2019-08-01,,,,,KENYAA
+NIA",,
+Missing date,,,,,,,
+Invalid country,2019-08-01,,,,,KENYAA,
+Uncommon filename extension,2019-08-10,,,,,,file.pdf.lck
+Unneccesary unicode (U+002D + U+00AD),2019-08-10,,978-­92-­9043-­823-­6,,,,

setup.py

@@ -13,7 +13,7 @@ install_requires = [
 
 setuptools.setup(
     name="csv-metadata-quality",
-    version="0.2.0",
+    version="0.2.1",
     author="Alan Orth",
     author_email="aorth@mjanja.ch",
     description="A simple, but opinionated CSV quality checking and fixing pipeline for CSVs in the DSpace ecosystem.",

tests/test_check.py

@@ -194,3 +194,24 @@ def test_check_valid_agrovoc():
     result = check.agrovoc(value, field_name)
 
     assert result == value
+
+
+def test_check_uncommon_filename_extension(capsys):
+    '''Test uncommon filename extension.'''
+
+    value = 'file.pdf.lck'
+
+    check.filename_extension(value)
+
+    captured = capsys.readouterr()
+    assert captured.out == f'Filename with uncommon extension: {value}\n'
+
+
+def test_check_common_filename_extension():
+    '''Test common filename extension.'''
+
+    value = 'file.pdf'
+
+    result = check.filename_extension(value)
+
+    assert result == value