mirror of
https://github.com/ilri/csv-metadata-quality.git
synced 2025-06-23 00:08:26 +02:00
Don't unnecessarily rewrite DataFrames for checks
By using df[column] = df[column].apply(check...) we were re-writing the DataFrame every time we returned from a check. We don't actuall y need to return a value at all, as the point of checks is to print a warning to the screen. In Python a "return" statement without a v ariable returns None. I haven't measured the impact of this, but I assume it will mean we are faster and use less memory.
This commit is contained in:
@ -105,7 +105,7 @@ def run(argv):
|
||||
df[column] = df[column].apply(fix.unnecessary_unicode)
|
||||
|
||||
# Check: suspicious characters
|
||||
df[column] = df[column].apply(check.suspicious_characters, field_name=column)
|
||||
df[column].apply(check.suspicious_characters, field_name=column)
|
||||
|
||||
# Fix: invalid and unnecessary multi-value separators
|
||||
df[column] = df[column].apply(fix.separators, field_name=column)
|
||||
@ -120,36 +120,36 @@ def run(argv):
|
||||
# Identify fields the user wants to validate against AGROVOC
|
||||
for field in args.agrovoc_fields.split(","):
|
||||
if column == field:
|
||||
df[column] = df[column].apply(check.agrovoc, field_name=column)
|
||||
df[column].apply(check.agrovoc, field_name=column)
|
||||
|
||||
# Check: invalid language
|
||||
match = re.match(r"^.*?language.*$", column)
|
||||
if match is not None:
|
||||
df[column] = df[column].apply(check.language)
|
||||
df[column].apply(check.language)
|
||||
|
||||
# Check: invalid ISSN
|
||||
match = re.match(r"^.*?issn.*$", column)
|
||||
if match is not None:
|
||||
df[column] = df[column].apply(check.issn)
|
||||
df[column].apply(check.issn)
|
||||
|
||||
# Check: invalid ISBN
|
||||
match = re.match(r"^.*?isbn.*$", column)
|
||||
if match is not None:
|
||||
df[column] = df[column].apply(check.isbn)
|
||||
df[column].apply(check.isbn)
|
||||
|
||||
# Check: invalid date
|
||||
match = re.match(r"^.*?(date|dcterms\.issued).*$", column)
|
||||
if match is not None:
|
||||
df[column] = df[column].apply(check.date, field_name=column)
|
||||
df[column].apply(check.date, field_name=column)
|
||||
|
||||
# Check: filename extension
|
||||
if column == "filename":
|
||||
df[column] = df[column].apply(check.filename_extension)
|
||||
df[column].apply(check.filename_extension)
|
||||
|
||||
# Check: SPDX license identifier
|
||||
match = re.match(r"dcterms\.license.*$", column)
|
||||
if match is not None:
|
||||
df[column] = df[column].apply(check.spdx_license_identifier)
|
||||
df[column].apply(check.spdx_license_identifier)
|
||||
|
||||
##
|
||||
# Perform some checks on rows so we can consider items as a whole rather
|
||||
|
@ -32,7 +32,7 @@ def issn(field):
|
||||
if not stdnum_issn.is_valid(value):
|
||||
print(f"{Fore.RED}Invalid ISSN: {Fore.RESET}{value}")
|
||||
|
||||
return field
|
||||
return
|
||||
|
||||
|
||||
def isbn(field):
|
||||
@ -55,7 +55,7 @@ def isbn(field):
|
||||
if not stdnum_isbn.is_valid(value):
|
||||
print(f"{Fore.RED}Invalid ISBN: {Fore.RESET}{value}")
|
||||
|
||||
return field
|
||||
return
|
||||
|
||||
|
||||
def date(field, field_name):
|
||||
@ -83,13 +83,13 @@ def date(field, field_name):
|
||||
f"{Fore.RED}Multiple dates not allowed ({field_name}): {Fore.RESET}{field}"
|
||||
)
|
||||
|
||||
return field
|
||||
return
|
||||
|
||||
try:
|
||||
# Check if date is valid YYYY format
|
||||
datetime.strptime(field, "%Y")
|
||||
|
||||
return field
|
||||
return
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
@ -97,7 +97,7 @@ def date(field, field_name):
|
||||
# Check if date is valid YYYY-MM format
|
||||
datetime.strptime(field, "%Y-%m")
|
||||
|
||||
return field
|
||||
return
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
@ -105,7 +105,7 @@ def date(field, field_name):
|
||||
# Check if date is valid YYYY-MM-DD format
|
||||
datetime.strptime(field, "%Y-%m-%d")
|
||||
|
||||
return field
|
||||
return
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
@ -113,11 +113,11 @@ def date(field, field_name):
|
||||
# Check if date is valid YYYY-MM-DDTHH:MM:SSZ format
|
||||
datetime.strptime(field, "%Y-%m-%dT%H:%M:%SZ")
|
||||
|
||||
return field
|
||||
return
|
||||
except ValueError:
|
||||
print(f"{Fore.RED}Invalid date ({field_name}): {Fore.RESET}{field}")
|
||||
|
||||
return field
|
||||
return
|
||||
|
||||
|
||||
def suspicious_characters(field, field_name):
|
||||
@ -151,7 +151,7 @@ def suspicious_characters(field, field_name):
|
||||
suspicious_character_msg = f"{Fore.YELLOW}Suspicious character ({field_name}): {Fore.RESET}{field_subset}"
|
||||
print(f"{suspicious_character_msg:1.80}")
|
||||
|
||||
return field
|
||||
return
|
||||
|
||||
|
||||
def language(field):
|
||||
@ -184,7 +184,7 @@ def language(field):
|
||||
else:
|
||||
print(f"{Fore.RED}Invalid language: {Fore.RESET}{value}")
|
||||
|
||||
return field
|
||||
return
|
||||
|
||||
|
||||
def agrovoc(field, field_name):
|
||||
@ -230,7 +230,7 @@ def agrovoc(field, field_name):
|
||||
if len(data["results"]) == 0:
|
||||
print(f"{Fore.RED}Invalid AGROVOC ({field_name}): {Fore.RESET}{value}")
|
||||
|
||||
return field
|
||||
return
|
||||
|
||||
|
||||
def filename_extension(field):
|
||||
@ -281,7 +281,7 @@ def filename_extension(field):
|
||||
if filename_extension_match is False:
|
||||
print(f"{Fore.YELLOW}Filename with uncommon extension: {Fore.RESET}{value}")
|
||||
|
||||
return field
|
||||
return
|
||||
|
||||
|
||||
def spdx_license_identifier(field):
|
||||
@ -301,4 +301,4 @@ def spdx_license_identifier(field):
|
||||
|
||||
pass
|
||||
|
||||
return field
|
||||
return
|
||||
|
@ -94,4 +94,4 @@ def correct_language(row):
|
||||
)
|
||||
|
||||
else:
|
||||
return language
|
||||
return
|
||||
|
Reference in New Issue
Block a user