alanorth/csv-metadata-quality
1
0
Fork 0
mirror of https://github.com/ilri/csv-metadata-quality.git synced 2025-05-09 14:46:00 +02:00
Code Releases Activity

Rework AGROVOC validation

Browse Source 
AGROVOC validation is now disabled by default, but can be enabled
on a field-by-field basis. For example, countries and regions are
also present in AGROVOC. Fields with these values can be enabled
using the new `--agrovoc-fields` option.

I reworked the script output to show the field name when printing
an invalid term so that the user knows in which field the term is.
This commit is contained in:
Alan Orth
2019-08-01 23:51:58 +03:00
parent 576b3a3638
commit bf876a046a
2 changed files with 12 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
csv_metadata_quality/app.py
View File

@ -7,6 +7,7 @@ import re
def parse_args(argv):
parser = argparse.ArgumentParser(description='Metadata quality checker and fixer.')
parser.add_argument('--agrovoc-fields', '-a', help='Comma-separated list of fields to validate against AGROVOC, for example: dc.subject,cg.coverage.country')
parser.add_argument('--input-file', '-i', help='Path to input file. Can be UTF-8 CSV or Excel XLSX.', required=True, type=argparse.FileType('r', encoding='UTF-8'))
parser.add_argument('--output-file', '-o', help='Path to output file (always CSV).', required=True, type=argparse.FileType('w', encoding='UTF-8'))
parser.add_argument('--unsafe-fixes', '-u', help='Perform unsafe fixes.', action='store_true')
@ -48,9 +49,11 @@ def run(argv):
df[column] = df[column].apply(fix.duplicates)
# Check: invalid AGROVOC subject
match = re.match(r'.*?dc\.subject.*$', column)
if match is not None:
df[column] = df[column].apply(check.agrovoc)
if args.agrovoc_fields:
# Identify fields the user wants to validate against AGROVOC
for field in args.agrovoc_fields.split(','):
if column == field:
df[column] = df[column].apply(check.agrovoc, field_name=column)
# Check: invalid language
match = re.match(r'^.*?language.*$', column)