mirror of
https://github.com/ilri/csv-metadata-quality.git
synced 2025-12-08 14:49:52 +01:00
Compare commits
5 Commits
9f5d2c2c4f
...
v0.4.4
| Author | SHA1 | Date | |
|---|---|---|---|
|
d76e72532a
|
|||
|
13980d2dde
|
|||
|
9aaaa62461
|
|||
|
a7fc5a246c
|
|||
|
7fb8acb866
|
@@ -4,10 +4,12 @@ All notable changes to this project will be documented in this file.
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## Unreleased
|
||||
## [0.4.4] - 2021-02-21
|
||||
### Added
|
||||
- Accept dates formatted in ISO 8601 extended with combined date and time, for
|
||||
example: 2020-08-31T11:04:56Z
|
||||
- Colorized output: red for errors, yellow for warnings and information, green
|
||||
for changes
|
||||
|
||||
### Updated
|
||||
- Run `poetry update` to update project dependencies
|
||||
|
||||
@@ -4,6 +4,7 @@ import signal
|
||||
import sys
|
||||
|
||||
import pandas as pd
|
||||
from colorama import Fore
|
||||
|
||||
import csv_metadata_quality.check as check
|
||||
import csv_metadata_quality.experimental as experimental
|
||||
@@ -77,7 +78,7 @@ def run(argv):
|
||||
if column == exclude and skip is False:
|
||||
skip = True
|
||||
if skip:
|
||||
print(f"Skipping {column}")
|
||||
print(f"{Fore.YELLOW}Skipping {Fore.RESET}{column}")
|
||||
|
||||
continue
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ from datetime import datetime, timedelta
|
||||
import pandas as pd
|
||||
import requests
|
||||
import requests_cache
|
||||
from colorama import Fore
|
||||
from pycountry import languages
|
||||
|
||||
|
||||
@@ -26,7 +27,7 @@ def issn(field):
|
||||
for value in field.split("||"):
|
||||
|
||||
if not issn.is_valid(value):
|
||||
print(f"Invalid ISSN: {value}")
|
||||
print(f"{Fore.RED}Invalid ISSN: {Fore.RESET}{value}")
|
||||
|
||||
return field
|
||||
|
||||
@@ -51,7 +52,7 @@ def isbn(field):
|
||||
for value in field.split("||"):
|
||||
|
||||
if not isbn.is_valid(value):
|
||||
print(f"Invalid ISBN: {value}")
|
||||
print(f"{Fore.RED}Invalid ISBN: {Fore.RESET}{value}")
|
||||
|
||||
return field
|
||||
|
||||
@@ -76,7 +77,9 @@ def separators(field, field_name):
|
||||
for value in field.split("||"):
|
||||
# Check if the current value is blank
|
||||
if value == "":
|
||||
print(f"Unnecessary multi-value separator ({field_name}): {field}")
|
||||
print(
|
||||
f"{Fore.RED}Unnecessary multi-value separator ({field_name}): {Fore.RESET}{field}"
|
||||
)
|
||||
|
||||
continue
|
||||
|
||||
@@ -85,7 +88,9 @@ def separators(field, field_name):
|
||||
|
||||
# Check if there was a match
|
||||
if match:
|
||||
print(f"Invalid multi-value separator ({field_name}): {field}")
|
||||
print(
|
||||
f"{Fore.RED}Invalid multi-value separator ({field_name}): {Fore.RESET}{field}"
|
||||
)
|
||||
|
||||
return field
|
||||
|
||||
@@ -102,7 +107,7 @@ def date(field, field_name):
|
||||
"""
|
||||
|
||||
if pd.isna(field):
|
||||
print(f"Missing date ({field_name}).")
|
||||
print(f"{Fore.RED}Missing date ({field_name}).{Fore.RESET}")
|
||||
|
||||
return
|
||||
|
||||
@@ -111,7 +116,9 @@ def date(field, field_name):
|
||||
|
||||
# We don't allow multi-value date fields
|
||||
if len(multiple_dates) > 1:
|
||||
print(f"Multiple dates not allowed ({field_name}): {field}")
|
||||
print(
|
||||
f"{Fore.RED}Multiple dates not allowed ({field_name}): {Fore.RESET}{field}"
|
||||
)
|
||||
|
||||
return field
|
||||
|
||||
@@ -145,7 +152,7 @@ def date(field, field_name):
|
||||
|
||||
return field
|
||||
except ValueError:
|
||||
print(f"Invalid date ({field_name}): {field}")
|
||||
print(f"{Fore.RED}Invalid date ({field_name}): {Fore.RESET}{field}")
|
||||
|
||||
return field
|
||||
|
||||
@@ -178,9 +185,7 @@ def suspicious_characters(field, field_name):
|
||||
# character and spanning enough of the rest to give a preview,
|
||||
# but not too much to cause the line to break in terminals with
|
||||
# a default of 80 characters width.
|
||||
suspicious_character_msg = (
|
||||
f"Suspicious character ({field_name}): {field_subset}"
|
||||
)
|
||||
suspicious_character_msg = f"{Fore.YELLOW}Suspicious character ({field_name}): {Fore.RESET}{field_subset}"
|
||||
print(f"{suspicious_character_msg:1.80}")
|
||||
|
||||
return field
|
||||
@@ -205,16 +210,16 @@ def language(field):
|
||||
# can check it against ISO 639-1 or ISO 639-3 accordingly.
|
||||
if len(value) == 2:
|
||||
if not languages.get(alpha_2=value):
|
||||
print(f"Invalid ISO 639-1 language: {value}")
|
||||
print(f"{Fore.RED}Invalid ISO 639-1 language: {Fore.RESET}{value}")
|
||||
|
||||
pass
|
||||
elif len(value) == 3:
|
||||
if not languages.get(alpha_3=value):
|
||||
print(f"Invalid ISO 639-3 language: {value}")
|
||||
print(f"{Fore.RED}Invalid ISO 639-3 language: {Fore.RESET}{value}")
|
||||
|
||||
pass
|
||||
else:
|
||||
print(f"Invalid language: {value}")
|
||||
print(f"{Fore.RED}Invalid language: {Fore.RESET}{value}")
|
||||
|
||||
return field
|
||||
|
||||
@@ -256,7 +261,7 @@ def agrovoc(field, field_name):
|
||||
|
||||
# check if there are any results
|
||||
if len(data["results"]) == 0:
|
||||
print(f"Invalid AGROVOC ({field_name}): {value}")
|
||||
print(f"{Fore.RED}Invalid AGROVOC ({field_name}): {Fore.RESET}{value}")
|
||||
|
||||
return field
|
||||
|
||||
@@ -309,6 +314,6 @@ def filename_extension(field):
|
||||
break
|
||||
|
||||
if filename_extension_match is False:
|
||||
print(f"Filename with uncommon extension: {value}")
|
||||
print(f"{Fore.YELLOW}Filename with uncommon extension: {Fore.RESET}{value}")
|
||||
|
||||
return field
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import pandas as pd
|
||||
from colorama import Fore
|
||||
|
||||
|
||||
def correct_language(row):
|
||||
@@ -10,10 +11,11 @@ def correct_language(row):
|
||||
language and returns the value in the language field if it does match.
|
||||
"""
|
||||
|
||||
from pycountry import languages
|
||||
import langid
|
||||
import re
|
||||
|
||||
import langid
|
||||
from pycountry import languages
|
||||
|
||||
# Initialize some variables at global scope so that we can set them in the
|
||||
# loop scope below and still be able to access them afterwards.
|
||||
language = ""
|
||||
@@ -83,12 +85,12 @@ def correct_language(row):
|
||||
detected_language = languages.get(alpha_2=langid_classification[0])
|
||||
if len(language) == 2 and language != detected_language.alpha_2:
|
||||
print(
|
||||
f"Possibly incorrect language {language} (detected {detected_language.alpha_2}): {title}"
|
||||
f"{Fore.YELLOW}Possibly incorrect language {language} (detected {detected_language.alpha_2}): {Fore.RESET}{title}"
|
||||
)
|
||||
|
||||
elif len(language) == 3 and language != detected_language.alpha_3:
|
||||
print(
|
||||
f"Possibly incorrect language {language} (detected {detected_language.alpha_3}): {title}"
|
||||
f"{Fore.YELLOW}Possibly incorrect language {language} (detected {detected_language.alpha_3}): {Fore.RESET}{title}"
|
||||
)
|
||||
|
||||
else:
|
||||
|
||||
@@ -2,6 +2,7 @@ import re
|
||||
from unicodedata import normalize
|
||||
|
||||
import pandas as pd
|
||||
from colorama import Fore
|
||||
|
||||
from csv_metadata_quality.util import is_nfc
|
||||
|
||||
@@ -29,7 +30,9 @@ def whitespace(field, field_name):
|
||||
match = re.findall(pattern, value)
|
||||
|
||||
if match:
|
||||
print(f"Removing excessive whitespace ({field_name}): {value}")
|
||||
print(
|
||||
f"{Fore.GREEN}Removing excessive whitespace ({field_name}): {Fore.RESET}{value}"
|
||||
)
|
||||
value = re.sub(pattern, " ", value)
|
||||
|
||||
# Save cleaned value
|
||||
@@ -62,7 +65,9 @@ def separators(field, field_name):
|
||||
for value in field.split("||"):
|
||||
# Check if the value is blank and skip it
|
||||
if value == "":
|
||||
print(f"Fixing unnecessary multi-value separator ({field_name}): {field}")
|
||||
print(
|
||||
f"{Fore.GREEN}Fixing unnecessary multi-value separator ({field_name}): {Fore.RESET}{field}"
|
||||
)
|
||||
|
||||
continue
|
||||
|
||||
@@ -71,7 +76,9 @@ def separators(field, field_name):
|
||||
match = re.findall(pattern, value)
|
||||
|
||||
if match:
|
||||
print(f"Fixing invalid multi-value separator ({field_name}): {value}")
|
||||
print(
|
||||
f"{Fore.RED}Fixing invalid multi-value separator ({field_name}): {Fore.RESET}{value}"
|
||||
)
|
||||
|
||||
value = re.sub(pattern, "||", value)
|
||||
|
||||
@@ -107,7 +114,7 @@ def unnecessary_unicode(field):
|
||||
match = re.findall(pattern, field)
|
||||
|
||||
if match:
|
||||
print(f"Removing unnecessary Unicode (U+200B): {field}")
|
||||
print(f"{Fore.GREEN}Removing unnecessary Unicode (U+200B): {Fore.RESET}{field}")
|
||||
field = re.sub(pattern, "", field)
|
||||
|
||||
# Check for replacement characters (U+FFFD)
|
||||
@@ -115,7 +122,7 @@ def unnecessary_unicode(field):
|
||||
match = re.findall(pattern, field)
|
||||
|
||||
if match:
|
||||
print(f"Removing unnecessary Unicode (U+FFFD): {field}")
|
||||
print(f"{Fore.GREEN}Removing unnecessary Unicode (U+FFFD): {Fore.RESET}{field}")
|
||||
field = re.sub(pattern, "", field)
|
||||
|
||||
# Check for no-break spaces (U+00A0)
|
||||
@@ -123,7 +130,9 @@ def unnecessary_unicode(field):
|
||||
match = re.findall(pattern, field)
|
||||
|
||||
if match:
|
||||
print(f"Replacing unnecessary Unicode (U+00A0): {field}")
|
||||
print(
|
||||
f"{Fore.GREEN}Replacing unnecessary Unicode (U+00A0): {Fore.RESET}{field}"
|
||||
)
|
||||
field = re.sub(pattern, " ", field)
|
||||
|
||||
# Check for soft hyphens (U+00AD), sometimes preceeded with a normal hyphen
|
||||
@@ -131,7 +140,9 @@ def unnecessary_unicode(field):
|
||||
match = re.findall(pattern, field)
|
||||
|
||||
if match:
|
||||
print(f"Replacing unnecessary Unicode (U+00AD): {field}")
|
||||
print(
|
||||
f"{Fore.GREEN}Replacing unnecessary Unicode (U+00AD): {Fore.RESET}{field}"
|
||||
)
|
||||
field = re.sub(pattern, "-", field)
|
||||
|
||||
return field
|
||||
@@ -156,7 +167,9 @@ def duplicates(field, field_name):
|
||||
if value not in new_values:
|
||||
new_values.append(value)
|
||||
else:
|
||||
print(f"Removing duplicate value ({field_name}): {value}")
|
||||
print(
|
||||
f"{Fore.GREEN}Removing duplicate value ({field_name}): {Fore.RESET}{value}"
|
||||
)
|
||||
|
||||
# Create a new field consisting of all values joined with "||"
|
||||
new_field = "||".join(new_values)
|
||||
@@ -189,7 +202,7 @@ def newlines(field):
|
||||
match = re.findall(r"\n", field)
|
||||
|
||||
if match:
|
||||
print(f"Removing newline: {field}")
|
||||
print(f"{Fore.GREEN}Removing newline: {Fore.RESET}{field}")
|
||||
field = field.replace("\n", "")
|
||||
|
||||
return field
|
||||
@@ -213,7 +226,9 @@ def comma_space(field, field_name):
|
||||
match = re.findall(r",\w", field)
|
||||
|
||||
if match:
|
||||
print(f"Adding space after comma ({field_name}): {field}")
|
||||
print(
|
||||
f"{Fore.GREEN}Adding space after comma ({field_name}): {Fore.RESET}{field}"
|
||||
)
|
||||
field = re.sub(r",(\w)", r", \1", field)
|
||||
|
||||
return field
|
||||
@@ -234,7 +249,7 @@ def normalize_unicode(field, field_name):
|
||||
|
||||
# Check if the current string is using normalized Unicode (NFC)
|
||||
if not is_nfc(field):
|
||||
print(f"Normalizing Unicode ({field_name}): {field}")
|
||||
print(f"{Fore.GREEN}Normalizing Unicode ({field_name}): {Fore.RESET}{field}")
|
||||
field = normalize("NFC", field)
|
||||
|
||||
return field
|
||||
|
||||
@@ -1 +1 @@
|
||||
VERSION = "0.4.3"
|
||||
VERSION = "0.4.4"
|
||||
|
||||
4
poetry.lock
generated
4
poetry.lock
generated
@@ -159,7 +159,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
|
||||
name = "colorama"
|
||||
version = "0.4.4"
|
||||
description = "Cross-platform colored terminal text."
|
||||
category = "dev"
|
||||
category = "main"
|
||||
optional = false
|
||||
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
|
||||
|
||||
@@ -765,7 +765,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
|
||||
[metadata]
|
||||
lock-version = "1.1"
|
||||
python-versions = "^3.8"
|
||||
content-hash = "63f2c6ef09652c4f8407660ff7b4690c8a07e5501eb8fc8c477f485de5888fcf"
|
||||
content-hash = "8c4ba410bbdc930d2d74f7864470a18827029a5697869833959708d7425460ae"
|
||||
|
||||
[metadata.files]
|
||||
agate = [
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "csv-metadata-quality"
|
||||
version = "0.4.3"
|
||||
version = "0.4.4"
|
||||
description="A simple, but opinionated CSV quality checking and fixing pipeline for CSVs in the DSpace ecosystem."
|
||||
authors = ["Alan Orth <alan.orth@gmail.com>"]
|
||||
license="GPL-3.0-only"
|
||||
@@ -16,6 +16,7 @@ requests = "^2.23.0"
|
||||
requests-cache = "^0.5.2"
|
||||
pycountry = "^19.8.18"
|
||||
langid = "^1.1.6"
|
||||
colorama = "^0.4.4"
|
||||
|
||||
[tool.poetry.dev-dependencies]
|
||||
pytest = "^6.1.1"
|
||||
|
||||
@@ -12,7 +12,7 @@ black==20.8b1; python_version >= "3.6"
|
||||
certifi==2020.12.5; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0"
|
||||
chardet==4.0.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0"
|
||||
click==7.1.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
|
||||
colorama==0.4.4; python_version >= "3.7" and python_full_version < "3.0.0" and sys_platform == "win32" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6") or sys_platform == "win32" and python_version >= "3.7" and python_full_version >= "3.5.0" and python_version < "4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6")
|
||||
colorama==0.4.4; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
|
||||
csvkit==1.0.5
|
||||
dbfread==2.0.7
|
||||
decorator==4.4.2; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4.0" or python_version >= "3.7" and python_version < "4.0" and python_full_version >= "3.2.0"
|
||||
@@ -30,29 +30,29 @@ langid==1.1.6
|
||||
leather==0.3.3
|
||||
mccabe==0.6.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0"
|
||||
mypy-extensions==0.4.3; python_version >= "3.6"
|
||||
numpy==1.20.0; python_version >= "3.7" and python_full_version >= "3.7.1"
|
||||
numpy==1.20.1; python_version >= "3.7" and python_full_version >= "3.7.1"
|
||||
openpyxl==3.0.6; python_version >= "3.6"
|
||||
packaging==20.9; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
|
||||
pandas==1.2.1; python_full_version >= "3.7.1"
|
||||
pandas==1.2.2; python_full_version >= "3.7.1"
|
||||
parsedatetime==2.6
|
||||
parso==0.8.1; python_version >= "3.7" and python_version < "4.0"
|
||||
pathspec==0.8.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
|
||||
pexpect==4.8.0; python_version >= "3.7" and python_version < "4.0" and sys_platform != "win32"
|
||||
pickleshare==0.7.5; python_version >= "3.7" and python_version < "4.0"
|
||||
pluggy==0.13.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
|
||||
prompt-toolkit==3.0.14; python_version >= "3.7" and python_version < "4.0" and python_full_version >= "3.6.1"
|
||||
prompt-toolkit==3.0.16; python_version >= "3.7" and python_version < "4.0" and python_full_version >= "3.6.1"
|
||||
ptyprocess==0.7.0; python_version >= "3.7" and python_version < "4.0" and sys_platform != "win32"
|
||||
py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
|
||||
pycodestyle==2.6.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0"
|
||||
pycountry==19.8.18
|
||||
pyflakes==2.2.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0"
|
||||
pygments==2.7.4; python_version >= "3.7" and python_version < "4.0"
|
||||
pygments==2.8.0; python_version >= "3.7" and python_version < "4.0"
|
||||
pyparsing==2.4.7; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
|
||||
pytest-clarity==0.3.0a0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
|
||||
pytest==6.2.2; python_version >= "3.6"
|
||||
python-dateutil==2.8.1; python_full_version >= "3.7.1"
|
||||
python-slugify==4.0.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0"
|
||||
python-stdnum==1.15
|
||||
python-stdnum==1.16
|
||||
pytimeparse==1.1.8
|
||||
pytz==2021.1; python_full_version >= "3.7.1"
|
||||
regex==2020.11.13; python_version >= "3.6"
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
certifi==2020.12.5; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0"
|
||||
chardet==4.0.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0"
|
||||
colorama==0.4.4; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
|
||||
idna==2.10; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0"
|
||||
langid==1.1.6
|
||||
numpy==1.20.0; python_version >= "3.7" and python_full_version >= "3.7.1"
|
||||
pandas==1.2.1; python_full_version >= "3.7.1"
|
||||
numpy==1.20.1; python_version >= "3.7" and python_full_version >= "3.7.1"
|
||||
pandas==1.2.2; python_full_version >= "3.7.1"
|
||||
pycountry==19.8.18
|
||||
python-dateutil==2.8.1; python_full_version >= "3.7.1"
|
||||
python-stdnum==1.15
|
||||
python-stdnum==1.16
|
||||
pytz==2021.1; python_full_version >= "3.7.1"
|
||||
requests-cache==0.5.2
|
||||
requests==2.25.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import pandas as pd
|
||||
from colorama import Fore
|
||||
|
||||
import csv_metadata_quality.check as check
|
||||
import csv_metadata_quality.experimental as experimental
|
||||
@@ -12,7 +13,7 @@ def test_check_invalid_issn(capsys):
|
||||
check.issn(value)
|
||||
|
||||
captured = capsys.readouterr()
|
||||
assert captured.out == f"Invalid ISSN: {value}\n"
|
||||
assert captured.out == f"{Fore.RED}Invalid ISSN: {Fore.RESET}{value}\n"
|
||||
|
||||
|
||||
def test_check_valid_issn():
|
||||
@@ -33,7 +34,7 @@ def test_check_invalid_isbn(capsys):
|
||||
check.isbn(value)
|
||||
|
||||
captured = capsys.readouterr()
|
||||
assert captured.out == f"Invalid ISBN: {value}\n"
|
||||
assert captured.out == f"{Fore.RED}Invalid ISBN: {Fore.RESET}{value}\n"
|
||||
|
||||
|
||||
def test_check_valid_isbn():
|
||||
@@ -56,7 +57,10 @@ def test_check_invalid_separators(capsys):
|
||||
check.separators(value, field_name)
|
||||
|
||||
captured = capsys.readouterr()
|
||||
assert captured.out == f"Invalid multi-value separator ({field_name}): {value}\n"
|
||||
assert (
|
||||
captured.out
|
||||
== f"{Fore.RED}Invalid multi-value separator ({field_name}): {Fore.RESET}{value}\n"
|
||||
)
|
||||
|
||||
|
||||
def test_check_unnecessary_separators(capsys):
|
||||
@@ -70,7 +74,8 @@ def test_check_unnecessary_separators(capsys):
|
||||
|
||||
captured = capsys.readouterr()
|
||||
assert (
|
||||
captured.out == f"Unnecessary multi-value separator ({field_name}): {field}\n"
|
||||
captured.out
|
||||
== f"{Fore.RED}Unnecessary multi-value separator ({field_name}): {Fore.RESET}{field}\n"
|
||||
)
|
||||
|
||||
|
||||
@@ -96,7 +101,7 @@ def test_check_missing_date(capsys):
|
||||
check.date(value, field_name)
|
||||
|
||||
captured = capsys.readouterr()
|
||||
assert captured.out == f"Missing date ({field_name}).\n"
|
||||
assert captured.out == f"{Fore.RED}Missing date ({field_name}).{Fore.RESET}\n"
|
||||
|
||||
|
||||
def test_check_multiple_dates(capsys):
|
||||
@@ -109,7 +114,10 @@ def test_check_multiple_dates(capsys):
|
||||
check.date(value, field_name)
|
||||
|
||||
captured = capsys.readouterr()
|
||||
assert captured.out == f"Multiple dates not allowed ({field_name}): {value}\n"
|
||||
assert (
|
||||
captured.out
|
||||
== f"{Fore.RED}Multiple dates not allowed ({field_name}): {Fore.RESET}{value}\n"
|
||||
)
|
||||
|
||||
|
||||
def test_check_invalid_date(capsys):
|
||||
@@ -122,7 +130,9 @@ def test_check_invalid_date(capsys):
|
||||
check.date(value, field_name)
|
||||
|
||||
captured = capsys.readouterr()
|
||||
assert captured.out == f"Invalid date ({field_name}): {value}\n"
|
||||
assert (
|
||||
captured.out == f"{Fore.RED}Invalid date ({field_name}): {Fore.RESET}{value}\n"
|
||||
)
|
||||
|
||||
|
||||
def test_check_valid_date():
|
||||
@@ -147,7 +157,10 @@ def test_check_suspicious_characters(capsys):
|
||||
check.suspicious_characters(value, field_name)
|
||||
|
||||
captured = capsys.readouterr()
|
||||
assert captured.out == f"Suspicious character ({field_name}): ˆt\n"
|
||||
assert (
|
||||
captured.out
|
||||
== f"{Fore.YELLOW}Suspicious character ({field_name}): {Fore.RESET}ˆt\n"
|
||||
)
|
||||
|
||||
|
||||
def test_check_valid_iso639_1_language():
|
||||
@@ -178,7 +191,9 @@ def test_check_invalid_iso639_1_language(capsys):
|
||||
check.language(value)
|
||||
|
||||
captured = capsys.readouterr()
|
||||
assert captured.out == f"Invalid ISO 639-1 language: {value}\n"
|
||||
assert (
|
||||
captured.out == f"{Fore.RED}Invalid ISO 639-1 language: {Fore.RESET}{value}\n"
|
||||
)
|
||||
|
||||
|
||||
def test_check_invalid_iso639_3_language(capsys):
|
||||
@@ -189,7 +204,9 @@ def test_check_invalid_iso639_3_language(capsys):
|
||||
check.language(value)
|
||||
|
||||
captured = capsys.readouterr()
|
||||
assert captured.out == f"Invalid ISO 639-3 language: {value}\n"
|
||||
assert (
|
||||
captured.out == f"{Fore.RED}Invalid ISO 639-3 language: {Fore.RESET}{value}\n"
|
||||
)
|
||||
|
||||
|
||||
def test_check_invalid_language(capsys):
|
||||
@@ -200,7 +217,7 @@ def test_check_invalid_language(capsys):
|
||||
check.language(value)
|
||||
|
||||
captured = capsys.readouterr()
|
||||
assert captured.out == f"Invalid language: {value}\n"
|
||||
assert captured.out == f"{Fore.RED}Invalid language: {Fore.RESET}{value}\n"
|
||||
|
||||
|
||||
def test_check_invalid_agrovoc(capsys):
|
||||
@@ -212,7 +229,10 @@ def test_check_invalid_agrovoc(capsys):
|
||||
check.agrovoc(value, field_name)
|
||||
|
||||
captured = capsys.readouterr()
|
||||
assert captured.out == f"Invalid AGROVOC ({field_name}): {value}\n"
|
||||
assert (
|
||||
captured.out
|
||||
== f"{Fore.RED}Invalid AGROVOC ({field_name}): {Fore.RESET}{value}\n"
|
||||
)
|
||||
|
||||
|
||||
def test_check_valid_agrovoc():
|
||||
@@ -234,7 +254,10 @@ def test_check_uncommon_filename_extension(capsys):
|
||||
check.filename_extension(value)
|
||||
|
||||
captured = capsys.readouterr()
|
||||
assert captured.out == f"Filename with uncommon extension: {value}\n"
|
||||
assert (
|
||||
captured.out
|
||||
== f"{Fore.YELLOW}Filename with uncommon extension: {Fore.RESET}{value}\n"
|
||||
)
|
||||
|
||||
|
||||
def test_check_common_filename_extension():
|
||||
@@ -262,7 +285,7 @@ def test_check_incorrect_iso_639_1_language(capsys):
|
||||
captured = capsys.readouterr()
|
||||
assert (
|
||||
captured.out
|
||||
== f"Possibly incorrect language {language} (detected en): {title}\n"
|
||||
== f"{Fore.YELLOW}Possibly incorrect language {language} (detected en): {Fore.RESET}{title}\n"
|
||||
)
|
||||
|
||||
|
||||
@@ -281,7 +304,7 @@ def test_check_incorrect_iso_639_3_language(capsys):
|
||||
captured = capsys.readouterr()
|
||||
assert (
|
||||
captured.out
|
||||
== f"Possibly incorrect language {language} (detected eng): {title}\n"
|
||||
== f"{Fore.YELLOW}Possibly incorrect language {language} (detected eng): {Fore.RESET}{title}\n"
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user