alanorth
/
csv-metadata-quality
mirror of https://github.com/ilri/csv-metadata-quality.git
1
0
Fork 0
Code Releases Activity

csv_metadata_quality/fix.py: Replace non-breaking spaces 

We should be replacing non-breaking spaces (U+00A0) with normal sp-
aces instead of removing them.
This commit is contained in:
Alan Orth 2019-10-01 16:55:04 +03:00
parent 1c75608d54
commit c42f8b4812
Signed by: alanorth
GPG Key ID: 0FB860CC9C45B1B9
1 changed files with 3 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
csv_metadata_quality/fix.py
View File

@ -74,10 +74,10 @@ def unnecessary_unicode(field):
Removes unnecessary Unicode characters like:
- Zero-width space (U+200B)
- Replacement character (U+FFFD)
- No-break space (U+00A0)
Replaces unnecessary Unicode characters like:
- Soft hyphen (U+00AD) → hyphen
- No-break space (U+00A0) → space
Return string with characters removed or replaced.
"""
@ -107,8 +107,8 @@ def unnecessary_unicode(field):
match = re.findall(pattern, field)
if match:
print(f"Removing unnecessary Unicode (U+00A0): {field}")
field = re.sub(pattern, "", field)
print(f"Replacing unnecessary Unicode (U+00A0): {field}")
field = re.sub(pattern, " ", field)
# Check for soft hyphens (U+00AD), sometimes preceeded with a normal hyphen
pattern = re.compile(r"\u002D*?\u00AD")