alanorth/csv-metadata-quality
1
0
Fork 0
mirror of https://github.com/ilri/csv-metadata-quality.git synced 2025-05-08 06:06:00 +02:00
Code Releases Activity

Output field name for more fixes and checks

Browse Source 
This helps identify which field has the error.
This commit is contained in:
Alan Orth
2020-01-16 12:35:11 +02:00
parent 40ba9bae6c
commit 28b5996aa6
5 changed files with 35 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
csv_metadata_quality/app.py
View File

@ -82,7 +82,7 @@ def run(argv):
continue
# Fix: whitespace
df[column] = df[column].apply(fix.whitespace)
df[column] = df[column].apply(fix.whitespace, field_name=column)
# Fix: newlines
if args.unsafe_fixes:
@ -104,19 +104,19 @@ def run(argv):
df[column] = df[column].apply(fix.unnecessary_unicode)
# Check: invalid multi-value separator
df[column] = df[column].apply(check.separators)
df[column] = df[column].apply(check.separators, field_name=column)
# Check: suspicious characters
df[column] = df[column].apply(check.suspicious_characters, field_name=column)
# Fix: invalid multi-value separator
if args.unsafe_fixes:
df[column] = df[column].apply(fix.separators)
df[column] = df[column].apply(fix.separators, field_name=column)
# Run whitespace fix again after fixing invalid separators
df[column] = df[column].apply(fix.whitespace)
df[column] = df[column].apply(fix.whitespace, field_name=column)
# Fix: duplicate metadata values
df[column] = df[column].apply(fix.duplicates)
df[column] = df[column].apply(fix.duplicates, field_name=column)
# Check: invalid AGROVOC subject
if args.agrovoc_fields:

4
csv_metadata_quality/check.py
View File

@ -51,7 +51,7 @@ def isbn(field):
return field
def separators(field):
def separators(field, field_name):
"""Check for invalid multi-value separators (ie "|" or "|||").
Prints the field with the invalid multi-value separator.
@ -70,7 +70,7 @@ def separators(field):
match = re.findall(r"^.*?\|.*$", value)
if match:
print(f"Invalid multi-value separator: {field}")
print(f"Invalid multi-value separator ({field_name}): {field}")
return field

12
csv_metadata_quality/fix.py
View File

@ -3,7 +3,7 @@ import re
import pandas as pd
def whitespace(field):
def whitespace(field, field_name):
"""Fix whitespace issues.
Return string with leading, trailing, and consecutive whitespace trimmed.
@ -26,7 +26,7 @@ def whitespace(field):
match = re.findall(pattern, value)
if match:
print(f"Removing excessive whitespace: {value}")
print(f"Removing excessive whitespace ({field_name}): {value}")
value = re.sub(pattern, " ", value)
# Save cleaned value
@ -38,7 +38,7 @@ def whitespace(field):
return new_field
def separators(field):
def separators(field, field_name):
"""Fix for invalid multi-value separators (ie "|")."""
# Skip fields with missing values
@ -55,7 +55,7 @@ def separators(field):
match = re.findall(pattern, value)
if match:
print(f"Fixing invalid multi-value separator: {value}")
print(f"Fixing invalid multi-value separator ({field_name}): {value}")
value = re.sub(pattern, "||", value)
@ -121,7 +121,7 @@ def unnecessary_unicode(field):
return field
def duplicates(field):
def duplicates(field, field_name):
"""Remove duplicate metadata values."""
# Skip fields with missing values
@ -140,7 +140,7 @@ def duplicates(field):
if value not in new_values:
new_values.append(value)
else:
print(f"Removing duplicate value: {value}")
print(f"Removing duplicate value ({field_name}): {value}")
# Create a new field consisting of all values joined with "||"
new_field = "||".join(new_values)