mirror of
https://github.com/ilri/csv-metadata-quality.git
synced 2025-05-17 09:53:53 +02:00
Add Unicode normalization
This will check all strings for un-normalized Unicode characters. Normalization is done using NFC. This includes tests and updated sample data (data/test.csv). See: https://withblue.ink/2019/03/11/why-you-need-to-normalize-unicode-strings.html
This commit is contained in:
@ -94,6 +94,11 @@ def run(argv):
|
||||
if match is not None:
|
||||
df[column] = df[column].apply(fix.comma_space, field_name=column)
|
||||
|
||||
# Fix: perform Unicode normalization (NFC) to convert decomposed
|
||||
# characters into their canonical forms.
|
||||
if args.unsafe_fixes:
|
||||
df[column] = df[column].apply(fix.normalize_unicode, field_name=column)
|
||||
|
||||
# Fix: unnecessary Unicode
|
||||
df[column] = df[column].apply(fix.unnecessary_unicode)
|
||||
|
||||
|
Reference in New Issue
Block a user