1
0
mirror of https://github.com/ilri/csv-metadata-quality.git synced 2024-11-16 02:57:04 +01:00
csv-metadata-quality/csv_metadata_quality/app.py

33 lines
1.1 KiB
Python
Raw Normal View History

import csv_metadata_quality.check as check
import csv_metadata_quality.fix as fix
import pandas as pd
import re
2019-07-27 22:09:16 +02:00
def main():
# Read all fields as strings so dates don't get converted from 1998 to 1998.0
#df = pd.read_csv('/home/aorth/Downloads/2019-07-26-Bioversity-Migration.csv', dtype=str)
#df = pd.read_csv('/tmp/quality.csv', dtype=str)
df = pd.read_csv('data/test.csv', dtype=str)
# Fix whitespace in all columns
for column in df.columns.values.tolist():
# Run whitespace fix on all columns
df[column] = df[column].apply(fix.whitespace)
# Run invalid multi-value separator check on all columns
df[column] = df[column].apply(check.separators)
if column == 'dc.identifier.issn':
df[column] = df[column].apply(check.issn)
if column == 'dc.identifier.isbn':
df[column] = df[column].apply(check.isbn)
# check if column is a date column like dc.date.issued
match = re.match(r'^.*?date.*$', column)
if match is not None:
df[column] = df[column].apply(check.date)
# Write
2019-07-26 22:14:37 +02:00
df.to_csv('/tmp/test.fixed.csv', index=False)