diff --git a/csv_metadata_quality/app.py b/csv_metadata_quality/app.py index 13dbcc0..7b5f3e7 100644 --- a/csv_metadata_quality/app.py +++ b/csv_metadata_quality/app.py @@ -1,6 +1,7 @@ import csv_metadata_quality.check as check import csv_metadata_quality.fix as fix import pandas as pd +import re def main(): # Read all fields as strings so dates don't get converted from 1998 to 1998.0 @@ -22,5 +23,10 @@ def main(): if column == 'dc.identifier.isbn': df[column] = df[column].apply(check.isbn) + # check if column is a date column like dc.date.issued + match = re.match(r'^.*?date.*$', column) + if match is not None: + df[column] = df[column].apply(check.date) + # Write df.to_csv('/tmp/test.fixed.csv', index=False) diff --git a/csv_metadata_quality/check.py b/csv_metadata_quality/check.py index 559d7e6..0527153 100755 --- a/csv_metadata_quality/check.py +++ b/csv_metadata_quality/check.py @@ -73,3 +73,54 @@ def separators(field): return field + + +def date(field): + """Check if a date is valid. + + In DSpace the issue date is usually 1990, 1990-01, or 1990-01-01, but it + could technically even include time as long as it is ISO8601. + + Also checks for other invalid cases like missing and multiple dates. + + Prints the date if invalid. + """ + from datetime import datetime + + if pd.isna(field): + print(f'Missing date.') + + return + + # Try to split multi-value field on "||" separator + multiple_dates = field.split('||') + + # We don't allow multi-value date fields + if len(multiple_dates) > 1: + print(f'Multiple dates not allowed: {field}') + + return field + + try: + # Check if date is valid YYYY format + datetime.strptime(field, '%Y') + + return field + except ValueError: + pass + + try: + # Check if date is valid YYYY-MM format + datetime.strptime(field, '%Y-%m') + + return field + except ValueError: + pass + + try: + # Check if date is valid YYYY-MM-DD format + datetime.strptime(field, '%Y-%m-%d') + + return field + except ValueError: + print(f'Invalid date: {field}') diff --git a/tests/test_check.py b/tests/test_check.py index 1f85eb9..d04b221 100644 --- a/tests/test_check.py +++ b/tests/test_check.py @@ -62,3 +62,46 @@ def test_check_valid_separators(): result = check.separators(value) assert result == value + + +def test_check_missing_date(capsys): + '''Test checking missing date.''' + + value = None + + check.date(value) + + captured = capsys.readouterr() + assert captured.out == f'Missing date.\n' + + +def test_check_multiple_dates(capsys): + '''Test checking multiple dates.''' + + value = '1990||1991' + + check.date(value) + + captured = capsys.readouterr() + assert captured.out == f'Multiple dates not allowed: {value}\n' + + +def test_check_invalid_date(capsys): + '''Test checking invalid ISO8601 date.''' + + value = '1990-0' + + check.date(value) + + captured = capsys.readouterr() + assert captured.out == f'Invalid date: {value}\n' + + +def test_check_valid_date(): + '''Test checking valid ISO8601 date.''' + + value = '1990' + + result = check.date(value) + + assert result == value