alanorth/csv-metadata-quality
1
0
Fork 0
mirror of https://github.com/ilri/csv-metadata-quality.git synced 2024-11-21 21:35:02 +01:00
Code Releases Activity

fix.py: Massive improvements

Browse Source 
Use Python's str.strip() instead of kludgy regular expressions and
use split/join to handle multi-value fields more cleanly.
This commit is contained in:
Alan Orth 2019-07-26 19:31:55 +03:00
parent 801870e0ba
commit ef5b8f7244
Signed by: alanorth
GPG Key ID: 0FB860CC9C45B1B9
1 changed files with 24 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
fix.py
View File

@ -2,7 +2,7 @@
import pandas as pd import pandas as pd
def fix_whitespace(value): def fix_whitespace(field):
"""Fix whitespace issues. """Fix whitespace issues.
Return string with leading, trailing, and consecutive whitespace trimmed. Return string with leading, trailing, and consecutive whitespace trimmed.
@ -10,62 +10,43 @@ def fix_whitespace(value):
import re import re
# Skip cells with missing values # Skip fields with missing values
if pd.isna(value): if pd.isna(field):
return return
# Try to split multi-value cells on "||" separator # Initialize an empty list to hold the cleaned values
#for value in cell.split('||'): values = list()
# Check for leading whitespace # Try to split multi-value field on "||" separator
pattern = re.compile(r'^\s+') for value in field.split('||'):
match = re.findall(pattern, value) # Strip leading and trailing whitespace
value = value.strip()
if len(match) > 0: # Replace excessive whitespace (>2) with one space
print('DEBUG: Leading whitespace') pattern = re.compile(r'\s{2,}')
value = re.sub(pattern, '', value) match = re.findall(pattern, value)
# Check for leading whitespace in multi-value cells if len(match) > 0:
# SOME VALUE|| ANOTHER VALUE print('DEBUG: Excessive whitespace')
pattern = re.compile(r'\|\|\s+') value = re.sub(pattern, ' ', value)
match = re.findall(pattern, value)
if len(match) > 0: # Save cleaned value
print('DEBUG: Leading whitespace in multi-value cell') values.append(value)
value = re.sub(pattern, '||', value)
# Check for trailing whitespace # Create a new field consisting of all values joined with "||"
pattern = re.compile(r'\s+$') new_field = '||'.join(values)
match = re.findall(pattern, value)
if len(match) > 0: return new_field
print('DEBUG: Trailing whitespace')
value = re.sub(pattern, '', value)
# Check for trailing whitespace in multi-value cells
# SOME VALUE ||ANOTHER VALUE
pattern = re.compile(r'\s+\|\|')
match = re.findall(pattern, value)
if len(match) > 0:
print('DEBUG: Trailing whitespace in multi-value cell')
value = re.sub(pattern, '||', value)
return value
# Read all fields as strings so dates don't get converted from 1998 to 1998.0 # Read all fields as strings so dates don't get converted from 1998 to 1998.0
df = pd.read_csv('/home/aorth/Downloads/2019-07-26-Bioversity-Migration.csv', dtype=str) #df = pd.read_csv('/home/aorth/Downloads/2019-07-26-Bioversity-Migration.csv', dtype=str)
#df = pd.read_csv('/tmp/quality.csv') #df = pd.read_csv('/tmp/quality.csv', dtype=str)
#df = pd.read_csv('/tmp/omg.csv') df = pd.read_csv('/tmp/omg.csv', dtype=str)
# Fix whitespace in all columns # Fix whitespace in all columns
for column in df.columns.values.tolist(): for column in df.columns.values.tolist():
print(column) print(f'DEBUG: {column}')
# Skip the id column
#if column == 'id':
# continue
df[column] = df[column].apply(fix_whitespace) df[column] = df[column].apply(fix_whitespace)