poetry.lock: run poetry update

From: https://github.com/spdx/license-list-data/blob/main/json/licenses.json

.drone.yml

@@ -1,3 +1,33 @@
+---
+kind: pipeline
+type: docker
+name: python311
+
+steps:
+- name: test
+  image: python:3.11-slim
+  commands:
+  - id
+  - python -V
+  - apt update && apt install -y gcc g++ libicu-dev pkg-config git
+  - python -m pip install poetry
+  - poetry install
+  - poetry run pytest
+  # Basic test
+  - poetry run csv-metadata-quality -i data/test.csv -o /tmp/test.csv
+  # Basic test with unsafe fixes
+  - poetry run csv-metadata-quality -i data/test.csv -o /tmp/test.csv -u
+  # Geography test
+  - poetry run csv-metadata-quality -i data/test-geography.csv -o /tmp/test.csv
+  # Geography test with unsafe fixes
+  - poetry run csv-metadata-quality -i data/test-geography.csv -o /tmp/test.csv -u
+  # Test with experimental checks
+  - poetry run csv-metadata-quality -i data/test.csv -o /tmp/test.csv -e
+  # Test with AGROVOC validation
+  - poetry run csv-metadata-quality -i data/test.csv -o /tmp/test.csv --agrovoc-fields dcterms.subject
+  # Test with AGROVOC validation (and dropping invalid)
+  - poetry run csv-metadata-quality -i data/test.csv -o /tmp/test.csv --agrovoc-fields dcterms.subject -d
+
 ---
 kind: pipeline
 type: docker
@@ -9,11 +39,24 @@ steps:
   commands:
   - id
   - python -V
-  - apt update && apt install -y gcc g++ libicu-dev pkg-config
-  - pip install -r requirements-dev.txt
-  - pytest
-  - python setup.py install
-  - csv-metadata-quality -i data/test.csv -o /tmp/test.csv -e -u --agrovoc-fields dcterms.subject,cg.coverage.country
+  - apt update && apt install -y gcc g++ libicu-dev pkg-config git
+  - python -m pip install poetry
+  - poetry install
+  - poetry run pytest
+  # Basic test
+  - poetry run csv-metadata-quality -i data/test.csv -o /tmp/test.csv
+  # Basic test with unsafe fixes
+  - poetry run csv-metadata-quality -i data/test.csv -o /tmp/test.csv -u
+  # Geography test
+  - poetry run csv-metadata-quality -i data/test-geography.csv -o /tmp/test.csv
+  # Geography test with unsafe fixes
+  - poetry run csv-metadata-quality -i data/test-geography.csv -o /tmp/test.csv -u
+  # Test with experimental checks
+  - poetry run csv-metadata-quality -i data/test.csv -o /tmp/test.csv -e
+  # Test with AGROVOC validation
+  - poetry run csv-metadata-quality -i data/test.csv -o /tmp/test.csv --agrovoc-fields dcterms.subject
+  # Test with AGROVOC validation (and dropping invalid)
+  - poetry run csv-metadata-quality -i data/test.csv -o /tmp/test.csv --agrovoc-fields dcterms.subject -d
 
 ---
 kind: pipeline
@@ -26,44 +69,23 @@ steps:
   commands:
   - id
   - python -V
-  - apt update && apt install -y gcc g++ libicu-dev pkg-config
-  - pip install -r requirements-dev.txt
-  - pytest
-  - python setup.py install
-  - csv-metadata-quality -i data/test.csv -o /tmp/test.csv -e -u --agrovoc-fields dcterms.subject,cg.coverage.country
-
----
-kind: pipeline
-type: docker
-name: python38
-
-steps:
-- name: test
-  image: python:3.8-slim
-  commands:
-  - id
-  - python -V
-  - apt update && apt install -y gcc g++ libicu-dev pkg-config
-  - pip install -r requirements-dev.txt
-  - pytest
-  - python setup.py install
-  - csv-metadata-quality -i data/test.csv -o /tmp/test.csv -e -u --agrovoc-fields dcterms.subject,cg.coverage.country
-
----
-kind: pipeline
-type: docker
-name: python37
-
-steps:
-- name: test
-  image: python:3.7-slim
-  commands:
-  - id
-  - python -V
-  - apt update && apt install -y gcc g++ libicu-dev pkg-config
-  - pip install -r requirements-dev.txt
-  - pytest
-  - python setup.py install
-  - csv-metadata-quality -i data/test.csv -o /tmp/test.csv -e -u --agrovoc-fields dcterms.subject,cg.coverage.country
+  - apt update && apt install -y gcc g++ libicu-dev pkg-config git
+  - python -m pip install poetry
+  - poetry install
+  - poetry run pytest
+  # Basic test
+  - poetry run csv-metadata-quality -i data/test.csv -o /tmp/test.csv
+  # Basic test with unsafe fixes
+  - poetry run csv-metadata-quality -i data/test.csv -o /tmp/test.csv -u
+  # Geography test
+  - poetry run csv-metadata-quality -i data/test-geography.csv -o /tmp/test.csv
+  # Geography test with unsafe fixes
+  - poetry run csv-metadata-quality -i data/test-geography.csv -o /tmp/test.csv -u
+  # Test with experimental checks
+  - poetry run csv-metadata-quality -i data/test.csv -o /tmp/test.csv -e
+  # Test with AGROVOC validation
+  - poetry run csv-metadata-quality -i data/test.csv -o /tmp/test.csv --agrovoc-fields dcterms.subject
+  # Test with AGROVOC validation (and dropping invalid)
+  - poetry run csv-metadata-quality -i data/test.csv -o /tmp/test.csv --agrovoc-fields dcterms.subject -d
 
 # vim: ts=2 sw=2 et

.github/workflows/python-app.yml

@@ -12,30 +12,34 @@ on:
 jobs:
   build:
 
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python 3.9
-      uses: actions/setup-python@v2
+    - uses: actions/checkout@v4
+    - name: Install poetry
+      run: pipx install poetry
+    - uses: actions/setup-python@v5
       with:
-        python-version: 3.9
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install flake8 pytest
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-        if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
+        python-version: '3.11'
+        cache: 'poetry'
+    - run: poetry install
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        poetry run flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+        poetry run flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
     - name: Test with pytest
-      run: |
-        pytest
+      run: poetry run pytest
     - name: Test CLI
       run: |
-        python setup.py install
-        csv-metadata-quality -i data/test.csv -o /tmp/test.csv -e -u --agrovoc-fields dcterms.subject,cg.coverage.country
+        # Basic test
+        poetry run csv-metadata-quality -i data/test.csv -o /tmp/test.csv
+        # Test with unsafe fixes
+        poetry run csv-metadata-quality -i data/test.csv -o /tmp/test.csv -u
+        # Test with experimental checks
+        poetry run csv-metadata-quality -i data/test.csv -o /tmp/test.csv -e
+        # Test with AGROVOC validation
+        poetry run csv-metadata-quality -i data/test.csv -o /tmp/test.csv --agrovoc-fields dcterms.subject
+        # Test with AGROVOC validation (and dropping invalid)
+        poetry run csv-metadata-quality -i data/test.csv -o /tmp/test.csv --agrovoc-fields dcterms.subject -d

CHANGELOG.md

@@ -4,6 +4,64 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+### Added
+- Ability to normalize DOIs to https://doi.org URI format
+
+### Fixed
+- Fixed regex so we don't run the invalid multi-value separator fix on
+`dcterms.bibliographicCitation` fields
+- Fixed regex so we run the comma space fix on `dcterms.bibliographicCitation`
+fields
+- Don't crash the country/region checker/fixer when a title field is missing
+
+### Changed
+- Don't run newline fix on description fields
+- Install requests-cache in main run() function instead of check.agrovoc() function so we only incur the overhead once
+- Use py3langid instead of langid, see: [How to make language detection with langid.py faster](https://adrien.barbaresi.eu/blog/language-detection-langid-py-faster.html)
+
+### Updated
+- Python dependencies, including Pandas 2.0.0 and [Arrow-backed dtypes](https://datapythonista.me/blog/pandas-20-and-the-arrow-revolution-part-i)
+- SPDX license list
+
+## [0.6.1] - 2023-02-23
+### Fixed
+- Missing region check should ignore subregion field, if it exists
+
+### Changed
+- Use SPDX license data from SPDX themselves instead of spdx-license-list
+because it is deprecated and outdated
+- Require Python 3.9+
+- Don't run `fix.separators()` on title or abstract fields
+- Don't run whitespace or newline fixes on abstract fields
+- Ignore some common non-SPDX licenses
+- Ignore `__description` suffix in filenames meant for SAFBuilder when checking
+for uncommon file extensions
+
+### Updated
+- Python dependencies
+
+## [0.6.0] - 2022-09-02
+### Changed
+- Perform fix for "unnecessary" Unicode characters after we try to fix encoding
+issues with ftfy
+- ftfy heuristics to use `is_bad()` instead of `sequence_weirdness()`
+- ftfy `fix_text()` to *not* change “smart quotes” to "ASCII quotes"
+
+### Updated
+- Python dependencies
+- Metadatata field exclude logic
+
+### Added
+- Ability to drop invalid AGROVOC values with `-d` when checking AGROVOC values
+with `-a <field.name>`
+- Ability to add missing UN M.49 regions when both country and region columns
+are present. Enable with `-u` (unsafe fixes) for now.
+
+### Removed
+- Support for reading Excel files (both `.xls` and `.xlsx`) as it was completely
+untested
+
 ## [0.5.0] - 2021-12-08
 ### Added
 - Ability to check for, and fix, "mojibake" characters using [ftfy](https://github.com/LuminosoInsight/python-ftfy)

MANIFEST.in

@@ -0,0 +1 @@
+include csv_metadata_quality/data/licenses.json

README.md

@@ -8,7 +8,7 @@
 
 A simple, but opinionated metadata quality checker and fixer designed to work with CSVs in the DSpace ecosystem (though it could theoretically work on any CSV that uses Dublin Core fields as columns). The implementation is essentially a pipeline of checks and fixes that begins with splitting multi-value fields on the standard DSpace "||" separator, trimming leading/trailing whitespace, and then proceeding to more specialized cases like ISSNs, ISBNs, languages, unnecessary Unicode, AGROVOC terms, etc.
 
-Requires Python 3.7.1 or greater (3.8+ recommended). CSV and Excel support comes from the [Pandas](https://pandas.pydata.org/) library, though your mileage may vary with Excel because this is much less tested.
+Requires Python 3.9 or greater. CSV support comes from the [Pandas](https://pandas.pydata.org/) library.
 
 If you use the DSpace CSV metadata quality checker please cite:
 
@@ -28,8 +28,10 @@ If you use the DSpace CSV metadata quality checker please cite:
 - Remove unnecessary Unicode like [non-breaking spaces](https://en.wikipedia.org/wiki/Non-breaking_space), [replacement characters](https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character), etc
 - Check for "suspicious" characters that indicate encoding or copy/paste issues, for example "foreˆt" should be "forêt"
 - Check for "mojibake" characters (and attempt to fix with `--unsafe-fixes`)
+- Check for countries with missing regions (and attempt to fix with `--unsafe-fixes`)
 - Remove duplicate metadata values
 - Check for duplicate items, using the title, type, and date issued as an indicator
+- [Normalize DOIs](https://www.crossref.org/documentation/member-setup/constructing-your-dois/) to https://doi.org URI format
 
 ## Installation
 The easiest way to install CSV Metadata Quality is with [poetry](https://python-poetry.org):
@@ -70,7 +72,7 @@ While it is *theoretically* possible for a single `|` character to be used legit
 This will also remove unnecessary trailing multi-value separators, for example `Kenya||Tanzania||`.
 
 ## Unsafe Fixes
-You can enable several "unsafe" fixes with the `--unsafe-fixes` option. Currently this will remove newlines, perform Unicode normalization, and attempt to fix "mojibake" characters.
+You can enable several "unsafe" fixes with the `--unsafe-fixes` option. Currently this will remove newlines, perform Unicode normalization, attempt to fix "mojibake" characters, and add missing UN M.49 regions.
 
 ### Newlines
 This is considered "unsafe" because some systems give special importance to vertical space and render it properly. DSpace does not support rendering newlines in its XMLUI and has, at times, suffered from parsing errors that cause the import process to fail if an input file had newlines. The `--unsafe-fixes` option strips Unix line feeds (U+000A).
@@ -91,6 +93,9 @@ Read more about [Unicode normalization](https://withblue.ink/2019/03/11/why-you-
 
 Pay special attention to the output of the script as well as the resulting file to make sure no new issues have been introduced. The ideal way to solve these issues is to avoid it in the first place. See [this guide about opening CSVs in UTF-8 format in Excel](https://www.itg.ias.edu/content/how-import-csv-file-uses-utf-8-character-encoding-0).
 
+### Countries With Missing Regions
+When an input file has both country and region columns we can check to see if the ISO 3166 country names have matching UN M.49 regions and add them when they are missing.
+
 ## AGROVOC Validation
 You can enable validation of metadata values in certain fields against the AGROVOC REST API with the `--agrovoc-fields` option. For example, in addition to agricultural subjects, many countries and regions are also present AGROVOC. Enable this validation by specifying a comma-separated list of fields:
 
@@ -121,9 +126,7 @@ This currently uses the [Python langid](https://github.com/saffsd/langid.py) lib
 - Better logging, for example with INFO, WARN, and ERR levels
 - Verbose, debug, or quiet options
 - Warn if an author is shorter than 3 characters?
-- Validate DOIs? Normalize to https://doi.org format? Or use just the DOI part: 10.1016/j.worlddev.2010.06.006
 - Warn if two items use the same file in `filename` column
-- Add an option to drop invalid AGROVOC subjects?
 - Add tests for application invocation, ie `tests/test_app.py`?
 - Validate ISSNs or journal titles against CrossRef API?
 - Add configurable field validation, like specify a field name and a validation file?
@@ -133,6 +136,7 @@ This currently uses the [Python langid](https://github.com/saffsd/langid.py) lib
   - Warn if item is Open Access, but missing a license
   - Warn if item has an ISSN but no journal title
   - Update journal titles from ISSN
+- Migrate from Pandas to Polars
 
 ## License
 This work is licensed under the [GPLv3](https://www.gnu.org/licenses/gpl-3.0.en.html).

csv_metadata_quality/app.py

@@ -1,11 +1,14 @@
 # SPDX-License-Identifier: GPL-3.0-only
 
 import argparse
+import os
 import re
 import signal
 import sys
+from datetime import timedelta
 
 import pandas as pd
+import requests_cache
 from colorama import Fore
 
 import csv_metadata_quality.check as check
@@ -21,6 +24,12 @@ def parse_args(argv):
         "-a",
         help="Comma-separated list of fields to validate against AGROVOC, for example: dcterms.subject,cg.coverage.country",
     )
+    parser.add_argument(
+        "--drop-invalid-agrovoc",
+        "-d",
+        help="After validating metadata values against AGROVOC, drop invalid values.",
+        action="store_true",
+    )
     parser.add_argument(
         "--experimental-checks",
         "-e",
@@ -30,7 +39,7 @@ def parse_args(argv):
     parser.add_argument(
         "--input-file",
         "-i",
-        help="Path to input file. Can be UTF-8 CSV or Excel XLSX.",
+        help="Path to input file. Must be a UTF-8 CSV.",
         required=True,
         type=argparse.FileType("r", encoding="UTF-8"),
     )
@@ -68,33 +77,50 @@ def run(argv):
     signal.signal(signal.SIGINT, signal_handler)
 
     # Read all fields as strings so dates don't get converted from 1998 to 1998.0
-    df = pd.read_csv(args.input_file, dtype=str)
+    df = pd.read_csv(args.input_file, dtype_backend="pyarrow", dtype="str")
+
+    # Check if the user requested to skip any fields
+    if args.exclude_fields:
+        # Split the list of excluded fields on ',' into a list. Note that the
+        # user should be careful to no include spaces here.
+        exclude = args.exclude_fields.split(",")
+    else:
+        exclude = []
+
+    # enable transparent request cache with thirty days expiry
+    expire_after = timedelta(days=30)
+    # Allow overriding the location of the requests cache, just in case we are
+    # running in an environment where we can't write to the current working di-
+    # rectory (for example from csv-metadata-quality-web).
+    REQUESTS_CACHE_DIR = os.environ.get("REQUESTS_CACHE_DIR", ".")
+    requests_cache.install_cache(
+        f"{REQUESTS_CACHE_DIR}/agrovoc-response-cache", expire_after=expire_after
+    )
+
+    # prune old cache entries
+    requests_cache.delete()
 
     for column in df.columns:
-        # Check if the user requested to skip any fields
-        if args.exclude_fields:
-            skip = False
-            # Split the list of excludes on ',' so we can test exact matches
-            # rather than fuzzy matches with regexes or "if word in string"
-            for exclude in args.exclude_fields.split(","):
-                if column == exclude and skip is False:
-                    skip = True
-            if skip:
-                print(f"{Fore.YELLOW}Skipping {Fore.RESET}{column}")
+        if column in exclude:
+            print(f"{Fore.YELLOW}Skipping {Fore.RESET}{column}")
 
-                continue
+            continue
 
-        # Fix: whitespace
-        df[column] = df[column].apply(fix.whitespace, field_name=column)
-
-        # Fix: newlines
         if args.unsafe_fixes:
-            df[column] = df[column].apply(fix.newlines, field_name=column)
+            # Skip whitespace and newline fixes on abstracts and descriptions
+            # because there are too many with legitimate multi-line metadata.
+            match = re.match(r"^.*?(abstract|description).*$", column)
+            if match is None:
+                # Fix: whitespace
+                df[column] = df[column].apply(fix.whitespace, field_name=column)
+
+                # Fix: newlines
+                df[column] = df[column].apply(fix.newlines, field_name=column)
 
         # Fix: missing space after comma. Only run on author and citation
         # fields for now, as this problem is mostly an issue in names.
         if args.unsafe_fixes:
-            match = re.match(r"^.*?(author|citation).*$", column)
+            match = re.match(r"^.*?(author|[Cc]itation).*$", column)
             if match is not None:
                 df[column] = df[column].apply(fix.comma_space, field_name=column)
 
@@ -103,9 +129,6 @@ def run(argv):
         if args.unsafe_fixes:
             df[column] = df[column].apply(fix.normalize_unicode, field_name=column)
 
-        # Fix: unnecessary Unicode
-        df[column] = df[column].apply(fix.unnecessary_unicode)
-
         # Check: suspicious characters
         df[column].apply(check.suspicious_characters, field_name=column)
 
@@ -115,20 +138,34 @@ def run(argv):
         else:
             df[column].apply(check.mojibake, field_name=column)
 
-        # Fix: invalid and unnecessary multi-value separators
-        df[column] = df[column].apply(fix.separators, field_name=column)
-        # Run whitespace fix again after fixing invalid separators
-        df[column] = df[column].apply(fix.whitespace, field_name=column)
+        # Fix: unnecessary Unicode
+        df[column] = df[column].apply(fix.unnecessary_unicode)
+
+        # Fix: normalize DOIs
+        match = re.match(r"^.*?identifier\.doi.*$", column)
+        if match is not None:
+            df[column] = df[column].apply(fix.normalize_dois)
+
+        # Fix: invalid and unnecessary multi-value separators. Skip the title
+        # and abstract fields because "|" is used to indicate something like
+        # a subtitle.
+        match = re.match(r"^.*?(abstract|[Cc]itation|title).*$", column)
+        if match is None:
+            df[column] = df[column].apply(fix.separators, field_name=column)
+            # Run whitespace fix again after fixing invalid separators
+            df[column] = df[column].apply(fix.whitespace, field_name=column)
 
         # Fix: duplicate metadata values
         df[column] = df[column].apply(fix.duplicates, field_name=column)
 
-        # Check: invalid AGROVOC subject
+        # Check: invalid AGROVOC subject and optionally drop them
         if args.agrovoc_fields:
             # Identify fields the user wants to validate against AGROVOC
             for field in args.agrovoc_fields.split(","):
                 if column == field:
-                    df[column].apply(check.agrovoc, field_name=column)
+                    df[column] = df[column].apply(
+                        check.agrovoc, field_name=column, drop=args.drop_invalid_agrovoc
+                    )
 
         # Check: invalid language
         match = re.match(r"^.*?language.*$", column)
@@ -192,19 +229,30 @@ def run(argv):
     # should rename column in this for loop...
     for column in df_transposed.columns:
         # Check: citation DOI
-        check.citation_doi(df_transposed[column])
+        check.citation_doi(df_transposed[column], exclude)
 
         # Check: title in citation
-        check.title_in_citation(df_transposed[column])
+        check.title_in_citation(df_transposed[column], exclude)
 
-        # Check: countries match regions
-        check.countries_match_regions(df_transposed[column])
+        if args.unsafe_fixes:
+            # Fix: countries match regions
+            df_transposed[column] = fix.countries_match_regions(
+                df_transposed[column], exclude
+            )
+        else:
+            # Check: countries match regions
+            check.countries_match_regions(df_transposed[column], exclude)
 
         if args.experimental_checks:
-            experimental.correct_language(df_transposed[column])
+            experimental.correct_language(df_transposed[column], exclude)
+
+    # Transpose the DataFrame back before writing. This is probably wasteful to
+    # do every time since we technically only need to do it if we've done the
+    # countries/regions fix above, but I can't think of another way for now.
+    df_transposed_back = df_transposed.T
 
     # Write
-    df.to_csv(args.output_file, index=False)
+    df_transposed_back.to_csv(args.output_file, index=False)
 
     # Close the input and output files before exiting
     args.input_file.close()

csv_metadata_quality/check.py

@@ -1,20 +1,18 @@
 # SPDX-License-Identifier: GPL-3.0-only
 
-import os
+import logging
 import re
-from datetime import datetime, timedelta
+from datetime import datetime
 
 import country_converter as coco
 import pandas as pd
 import requests
-import requests_cache
-import spdx_license_list
 from colorama import Fore
 from pycountry import languages
 from stdnum import isbn as stdnum_isbn
 from stdnum import issn as stdnum_issn
 
-from csv_metadata_quality.util import is_mojibake
+from csv_metadata_quality.util import is_mojibake, load_spdx_licenses
 
 
 def issn(field):
@@ -33,7 +31,6 @@ def issn(field):
 
     # Try to split multi-value field on "||" separator
     for value in field.split("||"):
-
         if not stdnum_issn.is_valid(value):
             print(f"{Fore.RED}Invalid ISSN: {Fore.RESET}{value}")
 
@@ -56,7 +53,6 @@ def isbn(field):
 
     # Try to split multi-value field on "||" separator
     for value in field.split("||"):
-
         if not stdnum_isbn.is_valid(value):
             print(f"{Fore.RED}Invalid ISBN: {Fore.RESET}{value}")
 
@@ -137,7 +133,7 @@ def suspicious_characters(field, field_name):
         return
 
     # List of suspicious characters, for example:  ́ˆ~`
-    suspicious_characters = ["\u00B4", "\u02C6", "\u007E", "\u0060"]
+    suspicious_characters = ["\u00b4", "\u02c6", "\u007e", "\u0060"]
 
     for character in suspicious_characters:
         # Find the position of the suspicious character in the string
@@ -173,7 +169,6 @@ def language(field):
 
     # Try to split multi-value field on "||" separator
     for value in field.split("||"):
-
         # After splitting, check if language value is 2 or 3 characters so we
         # can check it against ISO 639-1 or ISO 639-3 accordingly.
         if len(value) == 2:
@@ -188,7 +183,7 @@ def language(field):
     return
 
 
-def agrovoc(field, field_name):
+def agrovoc(field, field_name, drop):
     """Check subject terms against AGROVOC REST API.
 
     Function constructor expects the field as well as the field name because
@@ -206,22 +201,12 @@ def agrovoc(field, field_name):
     if pd.isna(field):
         return
 
-    # enable transparent request cache with thirty days expiry
-    expire_after = timedelta(days=30)
-    # Allow overriding the location of the requests cache, just in case we are
-    # running in an environment where we can't write to the current working di-
-    # rectory (for example from csv-metadata-quality-web).
-    REQUESTS_CACHE_DIR = os.environ.get("REQUESTS_CACHE_DIR", ".")
-    requests_cache.install_cache(
-        f"{REQUESTS_CACHE_DIR}/agrovoc-response-cache", expire_after=expire_after
-    )
-
-    # prune old cache entries
-    requests_cache.remove_expired_responses()
+    # Initialize an empty list to hold the validated AGROVOC values
+    values = []
 
     # Try to split multi-value field on "||" separator
     for value in field.split("||"):
-        request_url = "http://agrovoc.uniroma2.it/agrovoc/rest/v1/agrovoc/search"
+        request_url = "https://agrovoc.uniroma2.it/agrovoc/rest/v1/agrovoc/search"
         request_params = {"query": value}
 
         request = requests.get(request_url, params=request_params)
@@ -231,9 +216,25 @@ def agrovoc(field, field_name):
 
             # check if there are any results
             if len(data["results"]) == 0:
-                print(f"{Fore.RED}Invalid AGROVOC ({field_name}): {Fore.RESET}{value}")
+                if drop:
+                    print(
+                        f"{Fore.GREEN}Dropping invalid AGROVOC ({field_name}): {Fore.RESET}{value}"
+                    )
+                else:
+                    print(
+                        f"{Fore.RED}Invalid AGROVOC ({field_name}): {Fore.RESET}{value}"
+                    )
 
-    return
+                    # value is invalid AGROVOC, but we are not dropping
+                    values.append(value)
+            else:
+                # value is valid AGROVOC so save it
+                values.append(value)
+
+    # Create a new field consisting of all values joined with "||"
+    new_field = "||".join(values)
+
+    return new_field
 
 
 def filename_extension(field):
@@ -267,6 +268,11 @@ def filename_extension(field):
 
     # Iterate over all values
     for value in values:
+        # Strip filename descriptions that are meant for SAF Bundler, for
+        # example: Annual_Report_2020.pdf__description:Report
+        if "__description" in value:
+            value = value.split("__")[0]
+
         # Assume filename extension does not match
         filename_extension_match = False
 
@@ -293,13 +299,26 @@ def spdx_license_identifier(field):
     Prints the value if it is invalid.
     """
 
+    # List of common non-SPDX licenses to ignore
+    # See: https://ilri.github.io/cgspace-submission-guidelines/dcterms-license/dcterms-license.txt
+    ignore_licenses = {
+        "All rights reserved; no re-use allowed",
+        "All rights reserved; self-archive copy only",
+        "Copyrighted; Non-commercial educational use only",
+        "Copyrighted; Non-commercial use only",
+        "Copyrighted; all rights reserved",
+        "Other",
+    }
+
     # Skip fields with missing values
-    if pd.isna(field):
+    if pd.isna(field) or field in ignore_licenses:
         return
 
+    spdx_licenses = load_spdx_licenses()
+
     # Try to split multi-value field on "||" separator
     for value in field.split("||"):
-        if value not in spdx_license_list.LICENSES:
+        if value not in spdx_licenses:
             print(f"{Fore.YELLOW}Non-SPDX license identifier: {Fore.RESET}{value}")
 
     return
@@ -339,7 +358,7 @@ def duplicate_items(df):
 
     if items_count_unique < items_count_total:
         # Create a list to hold our items while we check for duplicates
-        items = list()
+        items = []
 
         for index, row in df.iterrows():
             item_title_type_date = f"{row[title_column_name]}{row[type_column_name]}{row[date_column_name]}"
@@ -371,13 +390,20 @@ def mojibake(field, field_name):
     return
 
 
-def citation_doi(row):
+def citation_doi(row, exclude):
     """Check for the scenario where an item has a DOI listed in its citation,
     but does not have a cg.identifier.doi field.
 
     Function prints a warning if the DOI field is missing, but there is a DOI
     in the citation.
     """
+    # Check if the user requested us to skip any DOI fields so we can
+    # just return before going any further.
+    for field in exclude:
+        match = re.match(r"^.*?doi.*$", field)
+        if match is not None:
+            return
+
     # Initialize some variables at global scope so that we can set them in the
     # loop scope below and still be able to access them afterwards.
     citation = ""
@@ -395,9 +421,10 @@ def citation_doi(row):
         if match is not None:
             return
 
-        # Get the name of the citation field
+        # Check if the current label is a citation field and make sure the user
+        # hasn't asked to skip it. If not, then set the citation.
         match = re.match(r"^.*?[cC]itation.*$", label)
-        if match is not None:
+        if match is not None and label not in exclude:
             citation = row[label]
 
     if citation != "":
@@ -413,7 +440,7 @@ def citation_doi(row):
     return
 
 
-def title_in_citation(row):
+def title_in_citation(row, exclude):
     """Check for the scenario where an item's title is missing from its cita-
     tion. This could mean that it is missing entirely, or perhaps just exists
     in a different format (whitespace, accents, etc).
@@ -435,12 +462,12 @@ def title_in_citation(row):
 
         # Find the name of the title column
         match = re.match(r"^(dc|dcterms)\.title.*$", label)
-        if match is not None:
+        if match is not None and label not in exclude:
             title = row[label]
 
         # Find the name of the citation column
         match = re.match(r"^.*?[cC]itation.*$", label)
-        if match is not None:
+        if match is not None and label not in exclude:
             citation = row[label]
 
     if citation != "":
@@ -450,7 +477,7 @@ def title_in_citation(row):
     return
 
 
-def countries_match_regions(row):
+def countries_match_regions(row, exclude):
     """Check for the scenario where an item has country coverage metadata, but
     does not have the corresponding region metadata. For example, an item that
     has country coverage "Kenya" should also have region "Eastern Africa" acc-
@@ -466,6 +493,15 @@ def countries_match_regions(row):
     region_column_name = ""
     title_column_name = ""
 
+    # Instantiate a CountryConverter() object here. According to the docs it is
+    # more performant to do that as opposed to calling coco.convert() directly
+    # because we don't need to re-load the country data with each iteration.
+    cc = coco.CountryConverter()
+
+    # Set logging to ERROR so country_converter's convert() doesn't print the
+    # "not found in regex" warning message to the screen.
+    logging.basicConfig(level=logging.ERROR)
+
     # Iterate over the labels of the current row's values to get the names of
     # the title and citation columns. Then we check if the title is present in
     # the citation.
@@ -475,9 +511,9 @@ def countries_match_regions(row):
         if match is not None:
             country_column_name = label
 
-        # Find the name of the region column
+        # Find the name of the region column, but make sure it's not subregion!
         match = re.match(r"^.*?region.*$", label)
-        if match is not None:
+        if match is not None and "sub" not in label:
             region_column_name = label
 
         # Find the name of the title column
@@ -485,6 +521,12 @@ def countries_match_regions(row):
         if match is not None:
             title_column_name = label
 
+    # Make sure the user has not asked to exclude any metadata fields. If so, we
+    # should return immediately.
+    column_names = [country_column_name, region_column_name, title_column_name]
+    if any(field in column_names for field in exclude):
+        return
+
     # Make sure we found the country and region columns
     if country_column_name != "" and region_column_name != "":
         # If we don't have any countries then we should return early before
@@ -497,25 +539,22 @@ def countries_match_regions(row):
         if row[region_column_name] is not None:
             regions = row[region_column_name].split("||")
         else:
-            regions = list()
-
-        # An empty list for our regions so we can keep track for all countries
-        missing_regions = list()
+            regions = []
 
         for country in countries:
             # Look up the UN M.49 regions for this country code. CoCo seems to
             # only list the direct region, ie Western Africa, rather than all
             # the parent regions ("Sub-Saharan Africa", "Africa", "World")
-            un_region = coco.convert(names=country, to="UNRegion")
+            un_region = cc.convert(names=country, to="UNRegion")
 
-            if un_region not in regions:
-                if un_region not in missing_regions:
-                    missing_regions.append(un_region)
-
-        if len(missing_regions) > 0:
-            for missing_region in missing_regions:
-                print(
-                    f"{Fore.YELLOW}Missing region ({missing_region}): {Fore.RESET}{row[title_column_name]}"
-                )
+            if un_region != "not found" and un_region not in regions:
+                try:
+                    print(
+                        f"{Fore.YELLOW}Missing region ({country} → {un_region}): {Fore.RESET}{row[title_column_name]}"
+                    )
+                except KeyError:
+                    print(
+                        f"{Fore.YELLOW}Missing region ({country} → {un_region}): {Fore.RESET}<title field not present>"
+                    )
 
     return

csv_metadata_quality/experimental.py

@@ -2,13 +2,13 @@
 
 import re
 
-import langid
 import pandas as pd
+import py3langid as langid
 from colorama import Fore
 from pycountry import languages
 
 
-def correct_language(row):
+def correct_language(row, exclude):
     """Analyze the text used in the title, abstract, and citation fields to pre-
     dict the language being used and compare it with the item's dc.language.iso
     field.
@@ -20,7 +20,7 @@ def correct_language(row):
     # Initialize some variables at global scope so that we can set them in the
     # loop scope below and still be able to access them afterwards.
     language = ""
-    sample_strings = list()
+    sample_strings = []
     title = None
 
     # Iterate over the labels of the current row's values. Before we transposed
@@ -39,7 +39,8 @@ def correct_language(row):
 
             language = row[label]
 
-        # Extract title if it is present
+        # Extract title if it is present (note that we don't allow excluding
+        # the title here because it complicates things).
         match = re.match(r"^.*?title.*$", label)
         if match is not None:
             title = row[label]
@@ -48,12 +49,12 @@ def correct_language(row):
 
         # Extract abstract if it is present
         match = re.match(r"^.*?abstract.*$", label)
-        if match is not None:
+        if match is not None and label not in exclude:
             sample_strings.append(row[label])
 
         # Extract citation if it is present
         match = re.match(r"^.*?[cC]itation.*$", label)
-        if match is not None:
+        if match is not None and label not in exclude:
             sample_strings.append(row[label])
 
     # Make sure language is not blank and is valid ISO 639-1/639-3 before proceeding with language prediction

csv_metadata_quality/fix.py

@@ -1,11 +1,13 @@
 # SPDX-License-Identifier: GPL-3.0-only
 
+import logging
 import re
 from unicodedata import normalize
 
+import country_converter as coco
 import pandas as pd
 from colorama import Fore
-from ftfy import fix_text
+from ftfy import TextFixerConfig, fix_text
 
 from csv_metadata_quality.util import is_mojibake, is_nfc
 
@@ -21,7 +23,7 @@ def whitespace(field, field_name):
         return
 
     # Initialize an empty list to hold the cleaned values
-    values = list()
+    values = []
 
     # Try to split multi-value field on "||" separator
     for value in field.split("||"):
@@ -62,7 +64,7 @@ def separators(field, field_name):
         return
 
     # Initialize an empty list to hold the cleaned values
-    values = list()
+    values = []
 
     # Try to split multi-value field on "||" separator
     for value in field.split("||"):
@@ -104,6 +106,7 @@ def unnecessary_unicode(field):
     Replaces unnecessary Unicode characters like:
         - Soft hyphen (U+00AD) → hyphen
         - No-break space (U+00A0) → space
+        - Thin space (U+2009) → space
 
     Return string with characters removed or replaced.
     """
@@ -148,6 +151,16 @@ def unnecessary_unicode(field):
         )
         field = re.sub(pattern, "-", field)
 
+    # Check for thin spaces (U+2009)
+    pattern = re.compile(r"\u2009")
+    match = re.findall(pattern, field)
+
+    if match:
+        print(
+            f"{Fore.GREEN}Replacing unnecessary Unicode (U+2009): {Fore.RESET}{field}"
+        )
+        field = re.sub(pattern, " ", field)
+
     return field
 
 
@@ -162,7 +175,7 @@ def duplicates(field, field_name):
     values = field.split("||")
 
     # Initialize an empty list to hold the de-duplicated values
-    new_values = list()
+    new_values = []
 
     # Iterate over all values
     for value in values:
@@ -269,9 +282,187 @@ def mojibake(field, field_name):
     if pd.isna(field):
         return field
 
+    # We don't want ftfy to change “smart quotes” to "ASCII quotes"
+    config = TextFixerConfig(uncurl_quotes=False)
+
     if is_mojibake(field):
         print(f"{Fore.GREEN}Fixing encoding issue ({field_name}): {Fore.RESET}{field}")
 
-        return fix_text(field)
+        return fix_text(field, config)
     else:
         return field
+
+
+def countries_match_regions(row, exclude):
+    """Check for the scenario where an item has country coverage metadata, but
+    does not have the corresponding region metadata. For example, an item that
+    has country coverage "Kenya" should also have region "Eastern Africa" acc-
+    ording to the UN M.49 classification scheme.
+
+    See: https://unstats.un.org/unsd/methodology/m49/
+
+    Return fixed string.
+    """
+    # Initialize some variables at global scope so that we can set them in the
+    # loop scope below and still be able to access them afterwards.
+    country_column_name = ""
+    region_column_name = ""
+    title_column_name = ""
+
+    # Instantiate a CountryConverter() object here. According to the docs it is
+    # more performant to do that as opposed to calling coco.convert() directly
+    # because we don't need to re-load the country data with each iteration.
+    cc = coco.CountryConverter()
+
+    # Set logging to ERROR so country_converter's convert() doesn't print the
+    # "not found in regex" warning message to the screen.
+    logging.basicConfig(level=logging.ERROR)
+
+    # Iterate over the labels of the current row's values to get the names of
+    # the title and citation columns. Then we check if the title is present in
+    # the citation.
+    for label in row.axes[0]:
+        # Find the name of the country column
+        match = re.match(r"^.*?country.*$", label)
+        if match is not None:
+            country_column_name = label
+
+        # Find the name of the region column, but make sure it's not subregion!
+        match = re.match(r"^.*?region.*$", label)
+        if match is not None and "sub" not in label:
+            region_column_name = label
+
+        # Find the name of the title column
+        match = re.match(r"^(dc|dcterms)\.title.*$", label)
+        if match is not None:
+            title_column_name = label
+
+    # Make sure the user has not asked to exclude any metadata fields. If so, we
+    # should return immediately.
+    column_names = [country_column_name, region_column_name, title_column_name]
+    if any(field in column_names for field in exclude):
+        return row
+
+    # Make sure we found the country and region columns
+    if country_column_name != "" and region_column_name != "":
+        # If we don't have any countries then we should return early before
+        # suggesting regions.
+        if row[country_column_name] is not None:
+            countries = row[country_column_name].split("||")
+        else:
+            return row
+
+        if row[region_column_name] is not None:
+            regions = row[region_column_name].split("||")
+        else:
+            regions = []
+
+        # An empty list for our regions so we can keep track for all countries
+        missing_regions = []
+
+        for country in countries:
+            # Look up the UN M.49 regions for this country code. CoCo seems to
+            # only list the direct region, ie Western Africa, rather than all
+            # the parent regions ("Sub-Saharan Africa", "Africa", "World")
+            un_region = cc.convert(names=country, to="UNRegion")
+
+            # Add the new un_region to regions if it is not "not found" and if
+            # it doesn't already exist in regions.
+            if un_region != "not found" and un_region not in regions:
+                if un_region not in missing_regions:
+                    try:
+                        print(
+                            f"{Fore.YELLOW}Adding missing region ({un_region}): {Fore.RESET}{row[title_column_name]}"
+                        )
+                    except KeyError:
+                        # If there is no title column in the CSV we will print
+                        # the fix without the title instead of crashing.
+                        print(
+                            f"{Fore.YELLOW}Adding missing region ({un_region}): {Fore.RESET}<title field not present>"
+                        )
+
+                    missing_regions.append(un_region)
+
+        if len(missing_regions) > 0:
+            # Add the missing regions back to the row, paying attention to whether
+            # or not the row's region column is None (aka null) or just an empty
+            # string (length would be 0).
+            if row[region_column_name] is not None and len(row[region_column_name]) > 0:
+                row[region_column_name] = (
+                    row[region_column_name] + "||" + "||".join(missing_regions)
+                )
+            else:
+                row[region_column_name] = "||".join(missing_regions)
+
+    return row
+
+
+def normalize_dois(field):
+    """Normalize DOIs.
+
+    DOIs are meant to be globally unique identifiers. They are case insensitive,
+    but in order to compare them robustly they should be normalized to a common
+    format:
+
+        - strip leading and trailing whitespace
+        - lowercase all ASCII characters
+        - convert all variations to https://doi.org/10.xxxx/xxxx URI format
+
+    Return string with normalized DOI.
+
+    See: https://www.crossref.org/documentation/member-setup/constructing-your-dois/
+    """
+
+    # Skip fields with missing values
+    if pd.isna(field):
+        return
+
+    # Try to split multi-value field on "||" separator
+    values = field.split("||")
+
+    # Initialize an empty list to hold the de-duplicated values
+    new_values = []
+
+    # Iterate over all values (most items will only have one DOI)
+    for value in values:
+        # Strip leading and trailing whitespace
+        new_value = value.strip()
+
+        new_value = new_value.lower()
+
+        # Convert to HTTPS
+        pattern = re.compile(r"^http://")
+        match = re.findall(pattern, new_value)
+
+        if match:
+            new_value = re.sub(pattern, "https://", new_value)
+
+        # Convert dx.doi.org to doi.org
+        pattern = re.compile(r"dx\.doi\.org")
+        match = re.findall(pattern, new_value)
+
+        if match:
+            new_value = re.sub(pattern, "doi.org", new_value)
+
+        # Replace values like doi: 10.11648/j.jps.20140201.14
+        pattern = re.compile(r"^doi: 10\.")
+        match = re.findall(pattern, new_value)
+
+        if match:
+            new_value = re.sub(pattern, "https://doi.org/10.", new_value)
+
+        # Replace values like 10.3390/foods12010115
+        pattern = re.compile(r"^10\.")
+        match = re.findall(pattern, new_value)
+
+        if match:
+            new_value = re.sub(pattern, "https://doi.org/10.", new_value)
+
+        if new_value != value:
+            print(f"{Fore.GREEN}Normalized DOI: {Fore.RESET}{value}")
+
+        new_values.append(new_value)
+
+    new_field = "||".join(new_values)
+
+    return new_field

csv_metadata_quality/util.py

@@ -1,6 +1,10 @@
 # SPDX-License-Identifier: GPL-3.0-only
 
-from ftfy.badness import sequence_weirdness
+
+import json
+import os
+
+from ftfy.badness import is_bad
 
 
 def is_nfc(field):
@@ -38,7 +42,7 @@ def is_mojibake(field):
 
     Return boolean.
     """
-    if not sequence_weirdness(field):
+    if not is_bad(field):
         # Nothing weird, should be okay
         return False
     try:
@@ -49,3 +53,13 @@ def is_mojibake(field):
     else:
         # Encodable as CP-1252, Mojibake alert level high
         return True
+
+
+def load_spdx_licenses():
+    """Returns a Python list of SPDX short license identifiers."""
+
+    with open(os.path.join(os.path.dirname(__file__), "data/licenses.json")) as f:
+        licenses = json.load(f)
+
+    # List comprehension to extract the license ID for each license
+    return [license["licenseId"] for license in licenses["licenses"]]

csv_metadata_quality/version.py

@@ -1,3 +1,3 @@
 # SPDX-License-Identifier: GPL-3.0-only
 
-VERSION = "0.5.0"
+VERSION = "0.6.1"

data/abstract-check.csv

@@ -0,0 +1,17 @@
+id,dc.title,dcterms.abstract
+1,Normal item,This is an abstract
+2,Leading whitespace, This is an abstract
+3,Trailing whitespace,This is an abstract 
+4,Consecutive whitespace,This is  an abstract
+5,Newline,"This
+is an abstract"
+6,Newline with leading whitespace," This
+ is an abstract"
+7,Newline with trailing whitespace,"This 
+is an abstract "
+8,Newline with consecutive whitespace,"This
+is an  abstract"
+9,Multiple newlines,"This
+is
+an
+abstract"

data/test-geography.csv

@@ -0,0 +1,13 @@
+dc.title,dcterms.issued,dcterms.type,dc.contributor.author,cg.coverage.country,cg.coverage.region
+No country,2022-09-01,Report,"Orth, Alan",,
+Matching country and region,2022-09-01,Report,"Orth, Alan",Kenya,Eastern Africa
+Missing region,2022-09-01,Report,"Orth, Alan",Kenya,
+Caribbean country with matching region,2022-09-01,Report,"Orth, Alan",Bahamas,Caribbean
+Caribbean country with no region,2022-09-01,Report,"Orth, Alan",Bahamas,
+Fake country with no region,2022-09-01,Report,"Orth, Alan",Yeah Baby,
+SE Asian country with matching region,2022-09-01,Report,"Orth, Alan",Cambodia,South-eastern Asia
+SE Asian country with no region,2022-09-01,Report,"Orth, Alan",Cambodia,
+Duplicate countries with matching region,2022-09-01,Report,"Orth, Alan",Kenya||Kenya,Eastern Africa
+Duplicate countries with missing regions,2022-09-01,Report,"Orth, Alan",Kenya||Kenya,
+Multiple countries with no regions,2022-09-01,Report,"Orth, Alan",Kenya||Bahamas,
+Multiple countries with mixed matching regions,2022-09-01,Report,"Orth, Alan",Kenya||Bahamas,Eastern Africa

data/test.csv

@@ -1,38 +1,42 @@
-dc.title,dcterms.issued,dc.identifier.issn,dc.identifier.isbn,dcterms.language,dcterms.subject,cg.coverage.country,filename,dcterms.license,dcterms.type,dcterms.bibliographicCitation,cg.identifier.doi,cg.coverage.region
- Leading space,2019-07-29,,,,,,,,,,,
-Trailing space ,2019-07-29,,,,,,,,,,,
-Excessive  space,2019-07-29,,,,,,,,,,,
-Miscellaenous ||whitespace | issues ,2019-07-29,,,,,,,,,,,
-Duplicate||Duplicate,2019-07-29,,,,,,,,,,,
-Invalid ISSN,2019-07-29,2321-2302,,,,,,,,,,
-Invalid ISBN,2019-07-29,,978-0-306-40615-6,,,,,,,,,
-Multiple valid ISSNs,2019-07-29,0378-5955||0024-9319,,,,,,,,,,
-Multiple valid ISBNs,2019-07-29,,99921-58-10-7||978-0-306-40615-7,,,,,,,,,
-Invalid date,2019-07-260,,,,,,,,,,,
-Multiple dates,2019-07-26||2019-01-10,,,,,,,,,,,
-Invalid multi-value separator,2019-07-29,0378-5955|0024-9319,,,,,,,,,,
-Unnecessary Unicode​,2019-07-29,,,,,,,,,,,
-Suspicious character||foreˆt,2019-07-29,,,,,,,,,,,
-Invalid ISO 639-1 (alpha 2) language,2019-07-29,,,jp,,,,,,,,
-Invalid ISO 639-3 (alpha 3) language,2019-07-29,,,chi,,,,,,,,
-Invalid language,2019-07-29,,,Span,,,,,,,,
-Invalid AGROVOC subject,2019-07-29,,,,FOREST,,,,,,,
+dc.title,dcterms.issued,dc.identifier.issn,dc.identifier.isbn,dcterms.language,dcterms.subject,cg.coverage.country,filename,dcterms.license,dcterms.type,dcterms.bibliographicCitation,cg.identifier.doi,cg.coverage.region,cg.coverage.subregion
+ Leading space,2019-07-29,,,,,,,,,,,,
+Trailing space ,2019-07-29,,,,,,,,,,,,
+Excessive  space,2019-07-29,,,,,,,,,,,,
+Miscellaenous ||whitespace | issues ,2019-07-29,,,,,,,,,,,,
+Duplicate||Duplicate,2019-07-29,,,,,,,,,,,,
+Invalid ISSN,2019-07-29,2321-2302,,,,,,,,,,,
+Invalid ISBN,2019-07-29,,978-0-306-40615-6,,,,,,,,,,
+Multiple valid ISSNs,2019-07-29,0378-5955||0024-9319,,,,,,,,,,,
+Multiple valid ISBNs,2019-07-29,,99921-58-10-7||978-0-306-40615-7,,,,,,,,,,
+Invalid date,2019-07-260,,,,,,,,,,,,
+Multiple dates,2019-07-26||2019-01-10,,,,,,,,,,,,
+Invalid multi-value separator,2019-07-29,0378-5955|0024-9319,,,,,,,,,,,
+Unnecessary Unicode​,2019-07-29,,,,,,,,,,,,
+Suspicious character||foreˆt,2019-07-29,,,,,,,,,,,,
+Invalid ISO 639-1 (alpha 2) language,2019-07-29,,,jp,,,,,,,,,
+Invalid ISO 639-3 (alpha 3) language,2019-07-29,,,chi,,,,,,,,,
+Invalid language,2019-07-29,,,Span,,,,,,,,,
+Invalid AGROVOC subject,2019-07-29,,,,LIVESTOCK||FOREST,,,,,,,,
 Newline (LF),2019-07-30,,,,"TANZA
-NIA",,,,,,,
-Missing date,,,,,,,,,,,,
-Invalid country,2019-08-01,,,,,KENYAA,,,,,,
-Uncommon filename extension,2019-08-10,,,,,,file.pdf.lck,,,,,
-Unneccesary unicode (U+002D + U+00AD),2019-08-10,,978-­92-­9043-­823-­6,,,,,,,,,
-"Missing space,after comma",2019-08-27,,,,,,,,,,,
-Incorrect ISO 639-1 language,2019-09-26,,,es,,,,,,,,
-Incorrect ISO 639-3 language,2019-09-26,,,spa,,,,,,,,
-Composéd Unicode,2020-01-14,,,,,,,,,,,
-Decomposéd Unicode,2020-01-14,,,,,,,,,,,
-Unnecessary multi-value separator,2021-01-03,0378-5955||,,,,,,,,,,
-Invalid SPDX license identifier,2021-03-11,,,,,,,CC-BY,,,,
-Duplicate Title,2021-03-17,,,,,,,,Report,,,
-Duplicate Title,2021-03-17,,,,,,,,Report,,,
-Mojibake,2021-03-18,,,,Publicaçao CIAT,,,,Report,,,
-"DOI in citation, but missing cg.identifier.doi",2021-10-06,,,,,,,,,"Orth, A. 2021. DOI in citation, but missing cg.identifier.doi. doi: 10.1186/1743-422X-9-218",,
-Title missing from citation,2021-12-05,,,,,,,,,"Orth, A. 2021. Title missing f rom citation.",,
-Country missing region,2021-12-08,,,,,Kenya,,,,,,
+NIA",,,,,,,,
+Missing date,,,,,,,,,,,,,
+Invalid country,2019-08-01,,,,,KENYAA,,,,,,,
+Uncommon filename extension,2019-08-10,,,,,,file.pdf.lck,,,,,,
+Unneccesary unicode (U+002D + U+00AD),2019-08-10,,978-­92-­9043-­823-­6,,,,,,,,,,
+"Missing space,after comma",2019-08-27,,,,,,,,,,,,
+Incorrect ISO 639-1 language,2019-09-26,,,es,,,,,,,,,
+Incorrect ISO 639-3 language,2019-09-26,,,spa,,,,,,,,,
+Composéd Unicode,2020-01-14,,,,,,,,,,,,
+Decomposéd Unicode,2020-01-14,,,,,,,,,,,,
+Unnecessary multi-value separator,2021-01-03,0378-5955||,,,,,,,,,,,
+Invalid SPDX license identifier,2021-03-11,,,,,,,CC-BY,,,,,
+Duplicate Title,2021-03-17,,,,,,,,Report,,,,
+Duplicate Title,2021-03-17,,,,,,,,Report,,,,
+Mojibake,2021-03-18,,,,Publicaçao CIAT,,,,Report,,,,
+"DOI in citation, but missing cg.identifier.doi",2021-10-06,,,,,,,,,"Orth, A. 2021. DOI in citation, but missing cg.identifier.doi. doi: 10.1186/1743-422X-9-218",,,
+Title missing from citation,2021-12-05,,,,,,,,,"Orth, A. 2021. Title missing f rom citation.",,,
+Country missing region,2021-12-08,,,,,Kenya,,,,,,,
+Subregion field shouldn’t trigger region checks,2022-12-07,,,,,Kenya,,,,,,Eastern Africa,Baringo
+DOI with HTTP and dx.doi.org,2024-04-23,,,,,,,,,,http://dx.doi.org/10.1016/j.envc.2023.100794,,
+DOI with colon,2024-04-23,,,,,,,,,,doi: 10.11648/j.jps.20140201.14,,
+Upper case bare DOI,2024-04-23,,,,,,,,,,10.19103/AS.2018.0043.16,,

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "csv-metadata-quality"
-version = "0.5.0"
+version = "0.6.1"
 description="A simple, but opinionated CSV quality checking and fixing pipeline for CSVs in the DSpace ecosystem."
 authors = ["Alan Orth <alan.orth@gmail.com>"]
 license="GPL-3.0-only"
@@ -11,29 +11,31 @@ homepage = "https://github.com/ilri/csv-metadata-quality"
 csv-metadata-quality = 'csv_metadata_quality.__main__:main'
 
 [tool.poetry.dependencies]
-python = "^3.7.1"
-pandas = "^1.0.4"
-python-stdnum = "^1.13"
-xlrd = "^1.2.0"
-requests = "^2.23.0"
-requests-cache = "~0.6.4"
-pycountry = "^19.8.18"
-langid = "^1.1.6"
-colorama = "^0.4.4"
-spdx-license-list = "^0.5.2"
-ftfy = "^5.9"
-SQLAlchemy = ">=1.3.3,<1.4.23"
-country-converter = "^0.7.4"
+python = "^3.9"
+pandas = {version = "^2.0.2", extras = ["feather", "performance"]}
+python-stdnum = "^1.18"
+requests = "^2.28.2"
+requests-cache = "^1.0.0"
+colorama = "^0.4.6"
+ftfy = "^6.1.1"
+country-converter = "~1.1.0"
+pycountry = "^23.12.7"
+py3langid = "^0.2.2"
 
-[tool.poetry.dev-dependencies]
-pytest = "^6.1.1"
-ipython = { version = "^7.18.1", python = "^3.7" }
-flake8 = "^3.8.4"
+[tool.poetry.group.dev.dependencies]
+pytest = "^7.2.1"
+flake8 = "^7.0.0"
 pytest-clarity = "^1.0.1"
-black = "^21.6b0"
-isort = "^5.5.4"
-csvkit = "^1.0.5"
+black = "^23.1.0"
+isort = "^5.12.0"
+csvkit = "^1.1.0"
+ipython = "^8.10.0"
+fixit = "^2.1.0"
 
 [build-system]
 requires = ["poetry>=0.12"]
 build-backend = "poetry.masonry.api"
+
+[tool.isort]
+profile = "black"
+line_length=88

renovate.json

@@ -0,0 +1,9 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:base"
+  ],
+  "pip_requirements": {
+      "enabled": false
+  }
+}

requirements-dev.txt

@@ -1,82 +1,82 @@
-agate-dbf==0.2.2
-agate-excel==0.2.5
-agate-sql==0.5.8
-agate==1.6.3
-appnope==0.1.2; python_version >= "3.7" and python_version < "4.0" and sys_platform == "darwin"
-atomicwrites==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" and sys_platform == "win32" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6") or sys_platform == "win32" and python_version >= "3.6" and python_full_version >= "3.4.0" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6")
-attrs==21.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
-babel==2.9.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0"
-backcall==0.2.0; python_version >= "3.7" and python_version < "4.0"
-black==21.12b0; python_full_version >= "3.6.2"
-certifi==2021.10.8; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
-charset-normalizer==2.0.9; python_full_version >= "3.6.0" and python_version >= "3.6"
-click==8.0.3; python_version >= "3.6" and python_full_version >= "3.6.2"
-colorama==0.4.4; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
-commonmark==0.9.1; python_full_version >= "3.6.2" and python_full_version < "4.0.0" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0")
-country-converter==0.7.4
-csvkit==1.0.6
-dbfread==2.0.7
-decorator==5.1.0; python_version >= "3.7" and python_version < "4.0"
-et-xmlfile==1.1.0; python_version >= "3.6"
-flake8==3.9.2; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0")
-ftfy==5.9; python_version >= "3.5"
-future==0.18.2; python_version >= "2.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0"
-greenlet==1.1.2; python_version >= "3" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3"
-idna==3.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
-importlib-metadata==4.8.2; python_full_version >= "3.6.2" and python_version < "3.8" and python_version >= "3.6" and (python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6") and (python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.8" or python_full_version >= "3.6.0" and python_version < "3.8" and python_version >= "3.6")
-iniconfig==1.1.1; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
-ipython==7.30.1; python_version >= "3.7" and python_version < "4.0"
-isodate==0.6.0
-isort==5.10.1; python_full_version >= "3.6.1" and python_version < "4.0"
-itsdangerous==2.0.1; python_version >= "3.6"
-jedi==0.18.1; python_version >= "3.7" and python_version < "4.0"
-langid==1.1.6
-leather==0.3.4
-matplotlib-inline==0.1.3; python_version >= "3.7" and python_version < "4.0"
-mccabe==0.6.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0"
-mypy-extensions==0.4.3; python_full_version >= "3.6.2"
-numpy==1.21.1
-olefile==0.46; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0"
-openpyxl==3.0.9; python_version >= "3.6"
-packaging==21.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
-pandas==1.3.4; python_full_version >= "3.7.1"
-parsedatetime==2.4
-parso==0.8.3; python_version >= "3.7" and python_version < "4.0"
-pathspec==0.9.0; python_full_version >= "3.6.2"
-pexpect==4.8.0; python_version >= "3.7" and python_version < "4.0" and sys_platform != "win32"
-pickleshare==0.7.5; python_version >= "3.7" and python_version < "4.0"
-platformdirs==2.4.0; python_version >= "3.6" and python_full_version >= "3.6.2"
-pluggy==1.0.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
-pprintpp==0.4.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0"
-prompt-toolkit==3.0.23; python_version >= "3.7" and python_version < "4.0" and python_full_version >= "3.6.2"
-ptyprocess==0.7.0; python_version >= "3.7" and python_version < "4.0" and sys_platform != "win32"
-py==1.11.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
-pycodestyle==2.7.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0"
-pycountry==19.8.18
-pyflakes==2.3.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0"
-pygments==2.10.0; python_version >= "3.7" and python_version < "4.0" and python_full_version >= "3.6.2" and python_full_version < "4.0.0" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0")
-pyparsing==3.0.6; python_version >= "3.6"
-pytest-clarity==1.0.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
-pytest==6.2.5; python_version >= "3.6"
-python-dateutil==2.8.2; python_full_version >= "3.7.1"
-python-slugify==5.0.2; python_version >= "3.6"
-python-stdnum==1.17
-pytimeparse==1.1.8
-pytz==2021.3; python_full_version >= "3.7.1"
-requests-cache==0.6.4; python_version >= "3.6"
-requests==2.26.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0")
-rich==10.15.2; python_full_version >= "3.6.2" and python_full_version < "4.0.0" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0")
-six==1.16.0; python_full_version >= "3.7.1" and python_version >= "3.6" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.3.0")
-spdx-license-list==0.5.2
-sqlalchemy==1.4.22; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0")
-text-unidecode==1.3; python_version >= "3.6"
-toml==0.10.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
-tomli==1.2.2; python_version >= "3.6" and python_full_version >= "3.6.2"
-traitlets==5.1.1; python_version >= "3.7" and python_version < "4.0"
-typed-ast==1.5.1; python_version < "3.8" and implementation_name == "cpython" and python_full_version >= "3.6.2" and python_version >= "3.6"
-typing-extensions==4.0.1
-url-normalize==1.4.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
-urllib3==1.26.7; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.6"
-wcwidth==0.2.5; python_version >= "3.7" and python_version < "4.0" and python_full_version >= "3.6.2"
-xlrd==1.2.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
-zipp==3.6.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.8" or python_full_version >= "3.6.0" and python_version < "3.8" and python_version >= "3.6"
+agate-dbf==0.2.2 ; python_version >= "3.9" and python_version < "4.0"
+agate-excel==0.2.5 ; python_version >= "3.9" and python_version < "4.0"
+agate-sql==0.5.9 ; python_version >= "3.9" and python_version < "4.0"
+agate==1.7.1 ; python_version >= "3.9" and python_version < "4.0"
+appdirs==1.4.4 ; python_version >= "3.9" and python_version < "4.0"
+appnope==0.1.3 ; python_version >= "3.9" and python_version < "4.0" and sys_platform == "darwin"
+asttokens==2.2.1 ; python_version >= "3.9" and python_version < "4.0"
+attrs==23.1.0 ; python_version >= "3.9" and python_version < "4.0"
+babel==2.12.1 ; python_version >= "3.9" and python_version < "4.0"
+backcall==0.2.0 ; python_version >= "3.9" and python_version < "4.0"
+black==23.3.0 ; python_version >= "3.9" and python_version < "4.0"
+cattrs==22.2.0 ; python_version >= "3.9" and python_version < "4.0"
+certifi==2022.12.7 ; python_version >= "3.9" and python_version < "4.0"
+charset-normalizer==3.1.0 ; python_version >= "3.9" and python_version < "4.0"
+click==8.1.3 ; python_version >= "3.9" and python_version < "4.0"
+colorama==0.4.6 ; python_version >= "3.9" and python_version < "4.0"
+country-converter==1.0.0 ; python_version >= "3.9" and python_version < "4.0"
+csvkit==1.1.1 ; python_version >= "3.9" and python_version < "4.0"
+dbfread==2.0.7 ; python_version >= "3.9" and python_version < "4.0"
+decorator==5.1.1 ; python_version >= "3.9" and python_version < "4.0"
+et-xmlfile==1.1.0 ; python_version >= "3.9" and python_version < "4.0"
+exceptiongroup==1.1.1 ; python_version >= "3.9" and python_version < "3.11"
+executing==1.2.0 ; python_version >= "3.9" and python_version < "4.0"
+flake8==6.0.0 ; python_version >= "3.9" and python_version < "4.0"
+ftfy==6.1.1 ; python_version >= "3.9" and python_version < "4"
+greenlet==2.0.2 ; python_version >= "3.9" and platform_machine == "aarch64" and python_version < "4.0" or python_version >= "3.9" and platform_machine == "ppc64le" and python_version < "4.0" or python_version >= "3.9" and platform_machine == "x86_64" and python_version < "4.0" or python_version >= "3.9" and platform_machine == "amd64" and python_version < "4.0" or python_version >= "3.9" and platform_machine == "AMD64" and python_version < "4.0" or python_version >= "3.9" and platform_machine == "win32" and python_version < "4.0" or python_version >= "3.9" and platform_machine == "WIN32" and python_version < "4.0"
+idna==3.4 ; python_version >= "3.9" and python_version < "4.0"
+iniconfig==2.0.0 ; python_version >= "3.9" and python_version < "4.0"
+ipython==8.13.1 ; python_version >= "3.9" and python_version < "4.0"
+isodate==0.6.1 ; python_version >= "3.9" and python_version < "4.0"
+isort==5.12.0 ; python_version >= "3.9" and python_version < "4.0"
+jedi==0.18.2 ; python_version >= "3.9" and python_version < "4.0"
+langid==1.1.6 ; python_version >= "3.9" and python_version < "4.0"
+leather==0.3.4 ; python_version >= "3.9" and python_version < "4.0"
+markdown-it-py==2.2.0 ; python_version >= "3.9" and python_version < "4.0"
+matplotlib-inline==0.1.6 ; python_version >= "3.9" and python_version < "4.0"
+mccabe==0.7.0 ; python_version >= "3.9" and python_version < "4.0"
+mdurl==0.1.2 ; python_version >= "3.9" and python_version < "4.0"
+mypy-extensions==1.0.0 ; python_version >= "3.9" and python_version < "4.0"
+numpy==1.24.3 ; python_version >= "3.9" and python_version < "4.0"
+olefile==0.46 ; python_version >= "3.9" and python_version < "4.0"
+openpyxl==3.1.2 ; python_version >= "3.9" and python_version < "4.0"
+packaging==23.1 ; python_version >= "3.9" and python_version < "4.0"
+pandas==2.0.1 ; python_version >= "3.9" and python_version < "4.0"
+parsedatetime==2.6 ; python_version >= "3.9" and python_version < "4.0"
+parso==0.8.3 ; python_version >= "3.9" and python_version < "4.0"
+pathspec==0.11.1 ; python_version >= "3.9" and python_version < "4.0"
+pexpect==4.8.0 ; python_version >= "3.9" and python_version < "4.0" and sys_platform != "win32"
+pickleshare==0.7.5 ; python_version >= "3.9" and python_version < "4.0"
+platformdirs==3.5.0 ; python_version >= "3.9" and python_version < "4.0"
+pluggy==1.0.0 ; python_version >= "3.9" and python_version < "4.0"
+pprintpp==0.4.0 ; python_version >= "3.9" and python_version < "4.0"
+prompt-toolkit==3.0.38 ; python_version >= "3.9" and python_version < "4.0"
+ptyprocess==0.7.0 ; python_version >= "3.9" and python_version < "4.0" and sys_platform != "win32"
+pure-eval==0.2.2 ; python_version >= "3.9" and python_version < "4.0"
+pyarrow==11.0.0 ; python_version >= "3.9" and python_version < "4.0"
+pycodestyle==2.10.0 ; python_version >= "3.9" and python_version < "4.0"
+pycountry @ git+https://github.com/alanorth/pycountry@iso-codes-4.13.0 ; python_version >= "3.9" and python_version < "4.0"
+pyflakes==3.0.1 ; python_version >= "3.9" and python_version < "4.0"
+pygments==2.15.1 ; python_version >= "3.9" and python_version < "4.0"
+pytest-clarity==1.0.1 ; python_version >= "3.9" and python_version < "4.0"
+pytest==7.3.1 ; python_version >= "3.9" and python_version < "4.0"
+python-dateutil==2.8.2 ; python_version >= "3.9" and python_version < "4.0"
+python-slugify==8.0.1 ; python_version >= "3.9" and python_version < "4.0"
+python-stdnum==1.18 ; python_version >= "3.9" and python_version < "4.0"
+pytimeparse==1.1.8 ; python_version >= "3.9" and python_version < "4.0"
+pytz==2023.3 ; python_version >= "3.9" and python_version < "4.0"
+requests-cache==0.9.8 ; python_version >= "3.9" and python_version < "4.0"
+requests==2.29.0 ; python_version >= "3.9" and python_version < "4.0"
+rich==13.3.5 ; python_version >= "3.9" and python_version < "4.0"
+six==1.16.0 ; python_version >= "3.9" and python_version < "4.0"
+sqlalchemy==1.4.48 ; python_version >= "3.9" and python_version < "4.0"
+stack-data==0.6.2 ; python_version >= "3.9" and python_version < "4.0"
+text-unidecode==1.3 ; python_version >= "3.9" and python_version < "4.0"
+tomli==2.0.1 ; python_version >= "3.9" and python_version < "3.11"
+traitlets==5.9.0 ; python_version >= "3.9" and python_version < "4.0"
+typing-extensions==4.5.0 ; python_version >= "3.9" and python_version < "3.10"
+tzdata==2023.3 ; python_version >= "3.9" and python_version < "4.0"
+url-normalize==1.4.3 ; python_version >= "3.9" and python_version < "4.0"
+urllib3==1.26.15 ; python_version >= "3.9" and python_version < "4.0"
+wcwidth==0.2.6 ; python_version >= "3.9" and python_version < "4"
+xlrd==2.0.1 ; python_version >= "3.9" and python_version < "4.0"

requirements.txt

@@ -1,201 +1,25 @@
-certifi==2021.10.8; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" \
-    --hash=sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569 \
-    --hash=sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872
-charset-normalizer==2.0.9; python_full_version >= "3.6.0" and python_version >= "3.6" \
-    --hash=sha256:b0b883e8e874edfdece9c28f314e3dd5badf067342e42fb162203335ae61aa2c \
-    --hash=sha256:1eecaa09422db5be9e29d7fc65664e6c33bd06f9ced7838578ba40d58bdf3721
-colorama==0.4.4; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") \
-    --hash=sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2 \
-    --hash=sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b
-country-converter==0.7.4 \
-    --hash=sha256:0291cc91c4a4efe7f128a11c8c6e4cb761f7fea7cde2517f8677c7c56da334d3
-ftfy==5.9; python_version >= "3.5" \
-    --hash=sha256:8c4fb2863c0b82eae2ab3cf353d9ade268dfbde863d322f78d6a9fd5cefb31e9
-greenlet==1.1.2; python_version >= "3" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3" \
-    --hash=sha256:58df5c2a0e293bf665a51f8a100d3e9956febfbf1d9aaf8c0677cf70218910c6 \
-    --hash=sha256:aec52725173bd3a7b56fe91bc56eccb26fbdff1386ef123abb63c84c5b43b63a \
-    --hash=sha256:833e1551925ed51e6b44c800e71e77dacd7e49181fdc9ac9a0bf3714d515785d \
-    --hash=sha256:aa5b467f15e78b82257319aebc78dd2915e4c1436c3c0d1ad6f53e47ba6e2713 \
-    --hash=sha256:40b951f601af999a8bf2ce8c71e8aaa4e8c6f78ff8afae7b808aae2dc50d4c40 \
-    --hash=sha256:95e69877983ea39b7303570fa6760f81a3eec23d0e3ab2021b7144b94d06202d \
-    --hash=sha256:356b3576ad078c89a6107caa9c50cc14e98e3a6c4874a37c3e0273e4baf33de8 \
-    --hash=sha256:8639cadfda96737427330a094476d4c7a56ac03de7265622fcf4cfe57c8ae18d \
-    --hash=sha256:97e5306482182170ade15c4b0d8386ded995a07d7cc2ca8f27958d34d6736497 \
-    --hash=sha256:e6a36bb9474218c7a5b27ae476035497a6990e21d04c279884eb10d9b290f1b1 \
-    --hash=sha256:abb7a75ed8b968f3061327c433a0fbd17b729947b400747c334a9c29a9af6c58 \
-    --hash=sha256:14d4f3cd4e8b524ae9b8aa567858beed70c392fdec26dbdb0a8a418392e71708 \
-    --hash=sha256:17ff94e7a83aa8671a25bf5b59326ec26da379ace2ebc4411d690d80a7fbcf23 \
-    --hash=sha256:9f3cba480d3deb69f6ee2c1825060177a22c7826431458c697df88e6aeb3caee \
-    --hash=sha256:fa877ca7f6b48054f847b61d6fa7bed5cebb663ebc55e018fda12db09dcc664c \
-    --hash=sha256:7cbd7574ce8e138bda9df4efc6bf2ab8572c9aff640d8ecfece1b006b68da963 \
-    --hash=sha256:903bbd302a2378f984aef528f76d4c9b1748f318fe1294961c072bdc7f2ffa3e \
-    --hash=sha256:049fe7579230e44daef03a259faa24511d10ebfa44f69411d99e6a184fe68073 \
-    --hash=sha256:dd0b1e9e891f69e7675ba5c92e28b90eaa045f6ab134ffe70b52e948aa175b3c \
-    --hash=sha256:7418b6bfc7fe3331541b84bb2141c9baf1ec7132a7ecd9f375912eca810e714e \
-    --hash=sha256:f9d29ca8a77117315101425ec7ec2a47a22ccf59f5593378fc4077ac5b754fce \
-    --hash=sha256:21915eb821a6b3d9d8eefdaf57d6c345b970ad722f856cd71739493ce003ad08 \
-    --hash=sha256:eff9d20417ff9dcb0d25e2defc2574d10b491bf2e693b4e491914738b7908168 \
-    --hash=sha256:32ca72bbc673adbcfecb935bb3fb1b74e663d10a4b241aaa2f5a75fe1d1f90aa \
-    --hash=sha256:f0214eb2a23b85528310dad848ad2ac58e735612929c8072f6093f3585fd342d \
-    --hash=sha256:b92e29e58bef6d9cfd340c72b04d74c4b4e9f70c9fa7c78b674d1fec18896dc4 \
-    --hash=sha256:fdcec0b8399108577ec290f55551d926d9a1fa6cad45882093a7a07ac5ec147b \
-    --hash=sha256:93f81b134a165cc17123626ab8da2e30c0455441d4ab5576eed73a64c025b25c \
-    --hash=sha256:1e12bdc622676ce47ae9abbf455c189e442afdde8818d9da983085df6312e7a1 \
-    --hash=sha256:8c790abda465726cfb8bb08bd4ca9a5d0a7bd77c7ac1ca1b839ad823b948ea28 \
-    --hash=sha256:f276df9830dba7a333544bd41070e8175762a7ac20350786b322b714b0e654f5 \
-    --hash=sha256:64e6175c2e53195278d7388c454e0b30997573f3f4bd63697f88d855f7a6a1fc \
-    --hash=sha256:b11548073a2213d950c3f671aa88e6f83cda6e2fb97a8b6317b1b5b33d850e06 \
-    --hash=sha256:9633b3034d3d901f0a46b7939f8c4d64427dfba6bbc5a36b1a67364cf148a1b0 \
-    --hash=sha256:eb6ea6da4c787111adf40f697b4e58732ee0942b5d3bd8f435277643329ba627 \
-    --hash=sha256:f3acda1924472472ddd60c29e5b9db0cec629fbe3c5c5accb74d6d6d14773478 \
-    --hash=sha256:e859fcb4cbe93504ea18008d1df98dee4f7766db66c435e4882ab35cf70cac43 \
-    --hash=sha256:00e44c8afdbe5467e4f7b5851be223be68adb4272f44696ee71fe46b7036a711 \
-    --hash=sha256:ec8c433b3ab0419100bd45b47c9c8551248a5aee30ca5e9d399a0b57ac04651b \
-    --hash=sha256:288c6a76705dc54fba69fbcb59904ae4ad768b4c768839b8ca5fdadec6dd8cfd \
-    --hash=sha256:8d2f1fb53a421b410751887eb4ff21386d119ef9cde3797bf5e7ed49fb51a3b3 \
-    --hash=sha256:166eac03e48784a6a6e0e5f041cfebb1ab400b394db188c48b3a84737f505b67 \
-    --hash=sha256:572e1787d1460da79590bf44304abbc0a2da944ea64ec549188fa84d89bba7ab \
-    --hash=sha256:be5f425ff1f5f4b3c1e33ad64ab994eed12fc284a6ea71c5243fd564502ecbe5 \
-    --hash=sha256:b1692f7d6bc45e3200844be0dba153612103db241691088626a33ff1f24a0d88 \
-    --hash=sha256:7227b47e73dedaa513cdebb98469705ef0d66eb5a1250144468e9c3097d6b59b \
-    --hash=sha256:7ff61ff178250f9bb3cd89752df0f1dd0e27316a8bd1465351652b1b4a4cdfd3 \
-    --hash=sha256:f70a9e237bb792c7cc7e44c531fd48f5897961701cdaa06cf22fc14965c496cf \
-    --hash=sha256:013d61294b6cd8fe3242932c1c5e36e5d1db2c8afb58606c5a67efce62c1f5fd \
-    --hash=sha256:e30f5ea4ae2346e62cedde8794a56858a67b878dd79f7df76a0767e356b1744a
-idna==3.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" \
-    --hash=sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff \
-    --hash=sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d
-importlib-metadata==4.8.2; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.8" or python_full_version >= "3.6.0" and python_version < "3.8" and python_version >= "3.6" \
-    --hash=sha256:53ccfd5c134223e497627b9815d5030edf77d2ed573922f7a0b8f8bb81a1c100 \
-    --hash=sha256:75bdec14c397f528724c1bfd9709d660b33a4d2e77387a3358f20b848bb5e5fb
-itsdangerous==2.0.1; python_version >= "3.6" \
-    --hash=sha256:5174094b9637652bdb841a3029700391451bd092ba3db90600dea710ba28e97c \
-    --hash=sha256:9e724d68fc22902a1435351f84c3fb8623f303fffcc566a4cb952df8c572cff0
-langid==1.1.6 \
-    --hash=sha256:044bcae1912dab85c33d8e98f2811b8f4ff1213e5e9a9e9510137b84da2cb293
-numpy==1.21.1 \
-    --hash=sha256:38e8648f9449a549a7dfe8d8755a5979b45b3538520d1e735637ef28e8c2dc50 \
-    --hash=sha256:fd7d7409fa643a91d0a05c7554dd68aa9c9bb16e186f6ccfe40d6e003156e33a \
-    --hash=sha256:a75b4498b1e93d8b700282dc8e655b8bd559c0904b3910b144646dbbbc03e062 \
-    --hash=sha256:1412aa0aec3e00bc23fbb8664d76552b4efde98fb71f60737c83efbac24112f1 \
-    --hash=sha256:e46ceaff65609b5399163de5893d8f2a82d3c77d5e56d976c8b5fb01faa6b671 \
-    --hash=sha256:c6a2324085dd52f96498419ba95b5777e40b6bcbc20088fddb9e8cbb58885e8e \
-    --hash=sha256:73101b2a1fef16602696d133db402a7e7586654682244344b8329cdcbbb82172 \
-    --hash=sha256:7a708a79c9a9d26904d1cca8d383bf869edf6f8e7650d85dbc77b041e8c5a0f8 \
-    --hash=sha256:95b995d0c413f5d0428b3f880e8fe1660ff9396dcd1f9eedbc311f37b5652e16 \
-    --hash=sha256:635e6bd31c9fb3d475c8f44a089569070d10a9ef18ed13738b03049280281267 \
-    --hash=sha256:4a3d5fb89bfe21be2ef47c0614b9c9c707b7362386c9a3ff1feae63e0267ccb6 \
-    --hash=sha256:8a326af80e86d0e9ce92bcc1e65c8ff88297de4fa14ee936cb2293d414c9ec63 \
-    --hash=sha256:791492091744b0fe390a6ce85cc1bf5149968ac7d5f0477288f78c89b385d9af \
-    --hash=sha256:0318c465786c1f63ac05d7c4dbcecd4d2d7e13f0959b01b534ea1e92202235c5 \
-    --hash=sha256:9a513bd9c1551894ee3d31369f9b07460ef223694098cf27d399513415855b68 \
-    --hash=sha256:91c6f5fc58df1e0a3cc0c3a717bb3308ff850abdaa6d2d802573ee2b11f674a8 \
-    --hash=sha256:978010b68e17150db8765355d1ccdd450f9fc916824e8c4e35ee620590e234cd \
-    --hash=sha256:9749a40a5b22333467f02fe11edc98f022133ee1bfa8ab99bda5e5437b831214 \
-    --hash=sha256:d7a4aeac3b94af92a9373d6e77b37691b86411f9745190d2c351f410ab3a791f \
-    --hash=sha256:d9e7912a56108aba9b31df688a4c4f5cb0d9d3787386b87d504762b6754fbb1b \
-    --hash=sha256:25b40b98ebdd272bc3020935427a4530b7d60dfbe1ab9381a39147834e985eac \
-    --hash=sha256:8a92c5aea763d14ba9d6475803fc7904bda7decc2a0a68153f587ad82941fec1 \
-    --hash=sha256:05a0f648eb28bae4bcb204e6fd14603de2908de982e761a2fc78efe0f19e96e1 \
-    --hash=sha256:f01f28075a92eede918b965e86e8f0ba7b7797a95aa8d35e1cc8821f5fc3ad6a \
-    --hash=sha256:88c0b89ad1cc24a5efbb99ff9ab5db0f9a86e9cc50240177a571fbe9c2860ac2 \
-    --hash=sha256:01721eefe70544d548425a07c80be8377096a54118070b8a62476866d5208e33 \
-    --hash=sha256:2d4d1de6e6fb3d28781c73fbde702ac97f03d79e4ffd6598b880b2d95d62ead4 \
-    --hash=sha256:dff4af63638afcc57a3dfb9e4b26d434a7a602d225b42d746ea7fe2edf1342fd
-pandas==1.3.4; python_full_version >= "3.7.1" \
-    --hash=sha256:9707bdc1ea9639c886b4d3be6e2a45812c1ac0c2080f94c31b71c9fa35556f9b \
-    --hash=sha256:c2f44425594ae85e119459bb5abb0748d76ef01d9c08583a667e3339e134218e \
-    --hash=sha256:372d72a3d8a5f2dbaf566a5fa5fa7f230842ac80f29a931fb4b071502cf86b9a \
-    --hash=sha256:d99d2350adb7b6c3f7f8f0e5dfb7d34ff8dd4bc0a53e62c445b7e43e163fce63 \
-    --hash=sha256:4acc28364863127bca1029fb72228e6f473bb50c32e77155e80b410e2068eeac \
-    --hash=sha256:c2646458e1dce44df9f71a01dc65f7e8fa4307f29e5c0f2f92c97f47a5bf22f5 \
-    --hash=sha256:5298a733e5bfbb761181fd4672c36d0c627320eb999c59c65156c6a90c7e1b4f \
-    --hash=sha256:22808afb8f96e2269dcc5b846decacb2f526dd0b47baebc63d913bf847317c8f \
-    --hash=sha256:b528e126c13816a4374e56b7b18bfe91f7a7f6576d1aadba5dee6a87a7f479ae \
-    --hash=sha256:fe48e4925455c964db914b958f6e7032d285848b7538a5e1b19aeb26ffaea3ec \
-    --hash=sha256:eaca36a80acaacb8183930e2e5ad7f71539a66805d6204ea88736570b2876a7b \
-    --hash=sha256:42493f8ae67918bf129869abea8204df899902287a7f5eaf596c8e54e0ac7ff4 \
-    --hash=sha256:a388960f979665b447f0847626e40f99af8cf191bce9dc571d716433130cb3a7 \
-    --hash=sha256:5ba0aac1397e1d7b654fccf263a4798a9e84ef749866060d19e577e927d66e1b \
-    --hash=sha256:f567e972dce3bbc3a8076e0b675273b4a9e8576ac629149cf8286ee13c259ae5 \
-    --hash=sha256:c1aa4de4919358c5ef119f6377bc5964b3a7023c23e845d9db7d9016fa0c5b1c \
-    --hash=sha256:dd324f8ee05925ee85de0ea3f0d66e1362e8c80799eb4eb04927d32335a3e44a \
-    --hash=sha256:d47750cf07dee6b55d8423471be70d627314277976ff2edd1381f02d52dbadf9 \
-    --hash=sha256:2d1dc09c0013d8faa7474574d61b575f9af6257ab95c93dcf33a14fd8d2c1bab \
-    --hash=sha256:10e10a2527db79af6e830c3d5842a4d60383b162885270f8cffc15abca4ba4a9 \
-    --hash=sha256:35c77609acd2e4d517da41bae0c11c70d31c87aae8dd1aabd2670906c6d2c143 \
-    --hash=sha256:003ba92db58b71a5f8add604a17a059f3068ef4e8c0c365b088468d0d64935fd \
-    --hash=sha256:a51528192755f7429c5bcc9e80832c517340317c861318fea9cea081b57c9afd \
-    --hash=sha256:a2aa18d3f0b7d538e21932f637fbfe8518d085238b429e4790a35e1e44a96ffc
-pycountry==19.8.18 \
-    --hash=sha256:3c57aa40adcf293d59bebaffbe60d8c39976fba78d846a018dc0c2ec9c6cb3cb
-python-dateutil==2.8.2; python_full_version >= "3.7.1" \
-    --hash=sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86 \
-    --hash=sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9
-python-stdnum==1.17 \
-    --hash=sha256:374e2b5e13912ccdbf50b0b23fca2c3e0531174805c32d74e145f37756328340 \
-    --hash=sha256:a46e6cf9652807314d369b654b255c86a59f93d18be2834f3d567ed1a346c547
-pytz==2021.3; python_full_version >= "3.7.1" \
-    --hash=sha256:3672058bc3453457b622aab7a1c3bfd5ab0bdae451512f6cf25f64ed37f5b87c \
-    --hash=sha256:acad2d8b20a1af07d4e4c9d2e9285c5ed9104354062f275f3fcd88dcef4f1326
-requests-cache==0.6.4; python_version >= "3.6" \
-    --hash=sha256:dd9120a4ab7b8128cba9b6b120d8b5560d566a3cd0f828cced3d3fd60a42ec40 \
-    --hash=sha256:1102daa13a804abe23fad62d694e7dee58d6063a35d94bf6e8c9821e22e5a78b
-requests==2.26.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0") \
-    --hash=sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24 \
-    --hash=sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7
-six==1.16.0; python_full_version >= "3.7.1" and python_version >= "3.6" \
-    --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254 \
-    --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926
-spdx-license-list==0.5.2 \
-    --hash=sha256:1b338470c7b403dbecceca563a316382c7977516128ca6c1e8f7078e3ed6e7b0 \
-    --hash=sha256:952996f72ab807972dc2278bb9b91e5294767211e51f09aad9c0e2ff5b82a31b
-sqlalchemy==1.4.22; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0") \
-    --hash=sha256:488608953385d6c127d2dcbc4b11f8d7f2f30b89f6bd27c01b042253d985cc2f \
-    --hash=sha256:5d856cc50fd26fc8dd04892ed5a5a3d7eeb914fea2c2e484183e2d84c14926e0 \
-    --hash=sha256:a00d9c6d3a8afe1d1681cd8a5266d2f0ed684b0b44bada2ca82403b9e8b25d39 \
-    --hash=sha256:5908ea6c652a050d768580d01219c98c071e71910ab8e7b42c02af4010608397 \
-    --hash=sha256:b7fb937c720847879c7402fe300cfdb2aeff22349fa4ea3651bca4e2d6555939 \
-    --hash=sha256:9bfe882d5a1bbde0245dca0bd48da0976bd6634cf2041d2fdf0417c5463e40e5 \
-    --hash=sha256:eedd76f135461cf237534a6dc0d1e0f6bb88a1dc193678fab48a11d223462da5 \
-    --hash=sha256:6a16c7c4452293da5143afa3056680db2d187b380b3ef4d470d4e29885720de3 \
-    --hash=sha256:44d23ea797a5e0be71bc5454b9ae99158ea0edc79e2393c6e9a2354de88329c0 \
-    --hash=sha256:a5e14cb0c0a4ac095395f24575a0e7ab5d1be27f5f9347f1762f21505e3ba9f1 \
-    --hash=sha256:bc34a007e604091ca3a4a057525efc4cefd2b7fe970f44d20b9cfa109ab1bddb \
-    --hash=sha256:756f5d2f5b92d27450167247fb574b09c4cd192a3f8c2e493b3e518a204ee543 \
-    --hash=sha256:9fcbb4b4756b250ed19adc5e28c005b8ed56fdb5c21efa24c6822c0575b4964d \
-    --hash=sha256:09dbb4bc01a734ccddbf188deb2a69aede4b3c153a72b6d5c6900be7fb2945b1 \
-    --hash=sha256:f028ef6a1d828bc754852a022b2160e036202ac8658a6c7d34875aafd14a9a15 \
-    --hash=sha256:68393d3fd31469845b6ba11f5b4209edbea0b58506be0e077aafbf9aa2e21e11 \
-    --hash=sha256:891927a49b2363a4199763a9d436d97b0b42c65922a4ea09025600b81a00d17e \
-    --hash=sha256:fd2102a8f8a659522719ed73865dff3d3cc76eb0833039dc473e0ad3041d04be \
-    --hash=sha256:4014978de28163cd8027434916a92d0f5bb1a3a38dff5e8bf8bff4d9372a9117 \
-    --hash=sha256:f814d80844969b0d22ea63663da4de5ca1c434cfbae226188901e5d368792c17 \
-    --hash=sha256:d09a760b0a045b4d799102ae7965b5491ccf102123f14b2a8cc6c01d1021a2d9 \
-    --hash=sha256:26daa429f039e29b1e523bf763bfab17490556b974c77b5ca7acb545b9230e9a \
-    --hash=sha256:12bac5fa1a6ea870bdccb96fe01610641dd44ebe001ed91ef7fcd980e9702db5 \
-    --hash=sha256:39b5d36ab71f73c068cdcf70c38075511de73616e6c7fdd112d6268c2704d9f5 \
-    --hash=sha256:5102b9face693e8b2db3b2539c7e1a5d9a5b4dc0d79967670626ffd2f710d6e6 \
-    --hash=sha256:c9373ef67a127799027091fa53449125351a8c943ddaa97bec4e99271dbb21f4 \
-    --hash=sha256:36a089dc604032d41343d86290ce85d4e6886012eea73faa88001260abf5ff81 \
-    --hash=sha256:b48148ceedfb55f764562e04c00539bb9ea72bf07820ca15a594a9a049ff6b0e \
-    --hash=sha256:1fdae7d980a2fa617d119d0dc13ecb5c23cc63a8b04ffcb5298f2c59d86851e9 \
-    --hash=sha256:ec1be26cdccd60d180359a527d5980d959a26269a2c7b1b327a1eea0cab37ed8
-typing-extensions==4.0.1; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.8" or python_full_version >= "3.6.0" and python_version < "3.8" and python_version >= "3.6" \
-    --hash=sha256:7f001e5ac290a0c0401508864c7ec868be4e701886d5b573a9528ed3973d9d3b \
-    --hash=sha256:4ca091dea149f945ec56afb48dae714f21e8692ef22a395223bcd328961b6a0e
-url-normalize==1.4.3; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" \
-    --hash=sha256:d23d3a070ac52a67b83a1c59a0e68f8608d1cd538783b401bc9de2c0fac999b2 \
-    --hash=sha256:ec3c301f04e5bb676d333a7fa162fa977ad2ca04b7e652bfc9fac4e405728eed
-urllib3==1.26.7; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.6" \
-    --hash=sha256:c4fdf4019605b6e5423637e01bc9fe4daef873709a7973e195ceba0a62bbc844 \
-    --hash=sha256:4987c65554f7a2dbf30c18fd48778ef124af6fab771a377103da0585e2336ece
-wcwidth==0.2.5; python_version >= "3.5" \
-    --hash=sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784 \
-    --hash=sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83
-xlrd==1.2.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") \
-    --hash=sha256:e551fb498759fa3a5384a94ccd4c3c02eb7c00ea424426e212ac0c57be9dfbde \
-    --hash=sha256:546eb36cee8db40c3eaa46c351e67ffee6eeb5fa2650b71bc4c758a29a1b29b2
-zipp==3.6.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.8" or python_full_version >= "3.6.0" and python_version < "3.8" and python_version >= "3.6" \
-    --hash=sha256:9fe5ea21568a0a70e50f273397638d39b03353731e6cbbb3fd8502a33fec40bc \
-    --hash=sha256:71c644c5369f4a6e07636f0aa966270449561fcea2e3d6747b8d23efaa9d7832
+appdirs==1.4.4 ; python_version >= "3.9" and python_version < "4.0"
+attrs==23.1.0 ; python_version >= "3.9" and python_version < "4.0"
+cattrs==22.2.0 ; python_version >= "3.9" and python_version < "4.0"
+certifi==2022.12.7 ; python_version >= "3.9" and python_version < "4.0"
+charset-normalizer==3.1.0 ; python_version >= "3.9" and python_version < "4.0"
+colorama==0.4.6 ; python_version >= "3.9" and python_version < "4.0"
+country-converter==1.0.0 ; python_version >= "3.9" and python_version < "4.0"
+exceptiongroup==1.1.1 ; python_version >= "3.9" and python_version < "3.11"
+ftfy==6.1.1 ; python_version >= "3.9" and python_version < "4"
+idna==3.4 ; python_version >= "3.9" and python_version < "4.0"
+langid==1.1.6 ; python_version >= "3.9" and python_version < "4.0"
+numpy==1.24.3 ; python_version >= "3.9" and python_version < "4.0"
+pandas==2.0.1 ; python_version >= "3.9" and python_version < "4.0"
+pyarrow==11.0.0 ; python_version >= "3.9" and python_version < "4.0"
+pycountry @ git+https://github.com/alanorth/pycountry@iso-codes-4.13.0 ; python_version >= "3.9" and python_version < "4.0"
+python-dateutil==2.8.2 ; python_version >= "3.9" and python_version < "4.0"
+python-stdnum==1.18 ; python_version >= "3.9" and python_version < "4.0"
+pytz==2023.3 ; python_version >= "3.9" and python_version < "4.0"
+requests-cache==0.9.8 ; python_version >= "3.9" and python_version < "4.0"
+requests==2.29.0 ; python_version >= "3.9" and python_version < "4.0"
+six==1.16.0 ; python_version >= "3.9" and python_version < "4.0"
+tzdata==2023.3 ; python_version >= "3.9" and python_version < "4.0"
+url-normalize==1.4.3 ; python_version >= "3.9" and python_version < "4.0"
+urllib3==1.26.15 ; python_version >= "3.9" and python_version < "4.0"
+wcwidth==0.2.6 ; python_version >= "3.9" and python_version < "4"

setup.cfg

@@ -1,6 +0,0 @@
-[isort]
-multi_line_output=3
-include_trailing_comma=True
-force_grid_wrap=0
-use_parentheses=True
-line_length=88

setup.py

@@ -1,37 +0,0 @@
-import setuptools
-
-with open("README.md", "r") as fh:
-    long_description = fh.read()
-
-install_requires = [
-    "pandas",
-    "python-stdnum",
-    "requests",
-    "requests-cache",
-    "pycountry",
-    "langid",
-]
-
-setuptools.setup(
-    name="csv-metadata-quality",
-    version="0.5.0",
-    author="Alan Orth",
-    author_email="aorth@mjanja.ch",
-    description="A simple, but opinionated CSV quality checking and fixing pipeline for CSVs in the DSpace ecosystem.",
-    license="GPLv3",
-    long_description=long_description,
-    long_description_content_type="text/markdown",
-    url="https://github.com/alanorth/csv-metadata-quality",
-    classifiers=[
-        "Programming Language :: Python :: 3.7",
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.9",
-        "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
-        "Operating System :: OS Independent",
-    ],
-    packages=["csv_metadata_quality"],
-    entry_points={
-        "console_scripts": ["csv-metadata-quality = csv_metadata_quality.__main__:main"]
-    },
-    install_requires=install_requires,
-)

tests/test_check.py

@@ -25,7 +25,7 @@ def test_check_valid_issn():
 
     result = check.issn(value)
 
-    assert result == None
+    assert result is None
 
 
 def test_check_invalid_isbn(capsys):
@@ -46,7 +46,7 @@ def test_check_valid_isbn():
 
     result = check.isbn(value)
 
-    assert result == None
+    assert result is None
 
 
 def test_check_missing_date(capsys):
@@ -102,7 +102,7 @@ def test_check_valid_date():
 
     result = check.date(value, field_name)
 
-    assert result == None
+    assert result is None
 
 
 def test_check_suspicious_characters(capsys):
@@ -128,7 +128,7 @@ def test_check_valid_iso639_1_language():
 
     result = check.language(value)
 
-    assert result == None
+    assert result is None
 
 
 def test_check_valid_iso639_3_language():
@@ -138,7 +138,7 @@ def test_check_valid_iso639_3_language():
 
     result = check.language(value)
 
-    assert result == None
+    assert result is None
 
 
 def test_check_invalid_iso639_1_language(capsys):
@@ -179,18 +179,41 @@ def test_check_invalid_language(capsys):
 
 
 def test_check_invalid_agrovoc(capsys):
-    """Test invalid AGROVOC subject."""
+    """Test invalid AGROVOC subject. Invalid values *will not* be dropped."""
 
-    value = "FOREST"
+    valid_agrovoc = "LIVESTOCK"
+    invalid_agrovoc = "FOREST"
+    value = f"{valid_agrovoc}||{invalid_agrovoc}"
     field_name = "dcterms.subject"
+    drop = False
 
-    check.agrovoc(value, field_name)
+    new_value = check.agrovoc(value, field_name, drop)
 
     captured = capsys.readouterr()
     assert (
         captured.out
-        == f"{Fore.RED}Invalid AGROVOC ({field_name}): {Fore.RESET}{value}\n"
+        == f"{Fore.RED}Invalid AGROVOC ({field_name}): {Fore.RESET}{invalid_agrovoc}\n"
     )
+    assert new_value == value
+
+
+def test_check_invalid_agrovoc_dropped(capsys):
+    """Test invalid AGROVOC subjects. Invalid values *will* be dropped."""
+
+    valid_agrovoc = "LIVESTOCK"
+    invalid_agrovoc = "FOREST"
+    value = f"{valid_agrovoc}||{invalid_agrovoc}"
+    field_name = "dcterms.subject"
+    drop = True
+
+    new_value = check.agrovoc(value, field_name, drop)
+
+    captured = capsys.readouterr()
+    assert (
+        captured.out
+        == f"{Fore.GREEN}Dropping invalid AGROVOC ({field_name}): {Fore.RESET}{invalid_agrovoc}\n"
+    )
+    assert new_value == valid_agrovoc
 
 
 def test_check_valid_agrovoc():
@@ -198,10 +221,11 @@ def test_check_valid_agrovoc():
 
     value = "FORESTS"
     field_name = "dcterms.subject"
+    drop = False
 
-    result = check.agrovoc(value, field_name)
+    result = check.agrovoc(value, field_name, drop)
 
-    assert result == None
+    assert result == "FORESTS"
 
 
 def test_check_uncommon_filename_extension(capsys):
@@ -225,7 +249,7 @@ def test_check_common_filename_extension():
 
     result = check.filename_extension(value)
 
-    assert result == None
+    assert result is None
 
 
 def test_check_incorrect_iso_639_1_language(capsys):
@@ -233,12 +257,13 @@ def test_check_incorrect_iso_639_1_language(capsys):
 
     title = "A randomised vaccine field trial in Kenya demonstrates protection against wildebeest-associated malignant catarrhal fever in cattle"
     language = "es"
+    exclude = []
 
     # Create a dictionary to mimic Pandas series
     row = {"dc.title": title, "dc.language.iso": language}
     series = pd.Series(row)
 
-    experimental.correct_language(series)
+    experimental.correct_language(series, exclude)
 
     captured = capsys.readouterr()
     assert (
@@ -252,12 +277,13 @@ def test_check_incorrect_iso_639_3_language(capsys):
 
     title = "A randomised vaccine field trial in Kenya demonstrates protection against wildebeest-associated malignant catarrhal fever in cattle"
     language = "spa"
+    exclude = []
 
     # Create a dictionary to mimic Pandas series
     row = {"dc.title": title, "dc.language.iso": language}
     series = pd.Series(row)
 
-    experimental.correct_language(series)
+    experimental.correct_language(series, exclude)
 
     captured = capsys.readouterr()
     assert (
@@ -271,14 +297,15 @@ def test_check_correct_iso_639_1_language():
 
     title = "A randomised vaccine field trial in Kenya demonstrates protection against wildebeest-associated malignant catarrhal fever in cattle"
     language = "en"
+    exclude = []
 
     # Create a dictionary to mimic Pandas series
     row = {"dc.title": title, "dc.language.iso": language}
     series = pd.Series(row)
 
-    result = experimental.correct_language(series)
+    result = experimental.correct_language(series, exclude)
 
-    assert result == None
+    assert result is None
 
 
 def test_check_correct_iso_639_3_language():
@@ -286,14 +313,15 @@ def test_check_correct_iso_639_3_language():
 
     title = "A randomised vaccine field trial in Kenya demonstrates protection against wildebeest-associated malignant catarrhal fever in cattle"
     language = "eng"
+    exclude = []
 
     # Create a dictionary to mimic Pandas series
     row = {"dc.title": title, "dc.language.iso": language}
     series = pd.Series(row)
 
-    result = experimental.correct_language(series)
+    result = experimental.correct_language(series, exclude)
 
-    assert result == None
+    assert result is None
 
 
 def test_check_valid_spdx_license_identifier():
@@ -303,7 +331,7 @@ def test_check_valid_spdx_license_identifier():
 
     result = check.spdx_license_identifier(license)
 
-    assert result == None
+    assert result is None
 
 
 def test_check_invalid_spdx_license_identifier(capsys):
@@ -311,7 +339,7 @@ def test_check_invalid_spdx_license_identifier(capsys):
 
     license = "CC-BY-SA"
 
-    result = check.spdx_license_identifier(license)
+    check.spdx_license_identifier(license)
 
     captured = capsys.readouterr()
     assert (
@@ -334,7 +362,7 @@ def test_check_duplicate_item(capsys):
     }
     df = pd.DataFrame(data=d)
 
-    result = check.duplicate_items(df)
+    check.duplicate_items(df)
 
     captured = capsys.readouterr()
     assert (
@@ -351,7 +379,7 @@ def test_check_no_mojibake():
 
     result = check.mojibake(field, field_name)
 
-    assert result == None
+    assert result is None
 
 
 def test_check_mojibake(capsys):
@@ -360,7 +388,7 @@ def test_check_mojibake(capsys):
     field = "CIAT Publicaçao"
     field_name = "dcterms.isPartOf"
 
-    result = check.mojibake(field, field_name)
+    check.mojibake(field, field_name)
 
     captured = capsys.readouterr()
     assert (
@@ -379,23 +407,25 @@ def test_check_doi_field():
     # the citation and a DOI field.
     d = {"cg.identifier.doi": doi, "dcterms.bibliographicCitation": citation}
     series = pd.Series(data=d)
+    exclude = []
 
-    result = check.citation_doi(series)
+    result = check.citation_doi(series, exclude)
 
-    assert result == None
+    assert result is None
 
 
 def test_check_doi_only_in_citation(capsys):
     """Test an item with a DOI in its citation, but no DOI field."""
 
     citation = "Orth, A. 2021. Testing all the things. doi: 10.1186/1743-422X-9-218"
+    exclude = []
 
     # Emulate a column in a transposed dataframe (which is just a series), with
     # an empty DOI field and a citation containing a DOI.
     d = {"cg.identifier.doi": None, "dcterms.bibliographicCitation": citation}
     series = pd.Series(data=d)
 
-    check.citation_doi(series)
+    check.citation_doi(series, exclude)
 
     captured = capsys.readouterr()
     assert (
@@ -409,15 +439,16 @@ def test_title_in_citation():
 
     title = "Testing all the things"
     citation = "Orth, A. 2021. Testing all the things."
+    exclude = []
 
     # Emulate a column in a transposed dataframe (which is just a series), with
     # the title and citation.
     d = {"dc.title": title, "dcterms.bibliographicCitation": citation}
     series = pd.Series(data=d)
 
-    result = check.title_in_citation(series)
+    result = check.title_in_citation(series, exclude)
 
-    assert result == None
+    assert result is None
 
 
 def test_title_not_in_citation(capsys):
@@ -425,13 +456,14 @@ def test_title_not_in_citation(capsys):
 
     title = "Testing all the things"
     citation = "Orth, A. 2021. Testing all teh things."
+    exclude = []
 
     # Emulate a column in a transposed dataframe (which is just a series), with
     # the title and citation.
     d = {"dc.title": title, "dcterms.bibliographicCitation": citation}
     series = pd.Series(data=d)
 
-    check.title_in_citation(series)
+    check.title_in_citation(series, exclude)
 
     captured = capsys.readouterr()
     assert (
@@ -445,14 +477,15 @@ def test_country_matches_region():
 
     country = "Kenya"
     region = "Eastern Africa"
+    exclude = []
 
     # Emulate a column in a transposed dataframe (which is just a series)
     d = {"cg.coverage.country": country, "cg.coverage.region": region}
     series = pd.Series(data=d)
 
-    result = check.countries_match_regions(series)
+    result = check.countries_match_regions(series, exclude)
 
-    assert result == None
+    assert result is None
 
 
 def test_country_not_matching_region(capsys):
@@ -462,6 +495,7 @@ def test_country_not_matching_region(capsys):
     country = "Kenya"
     region = ""
     missing_region = "Eastern Africa"
+    exclude = []
 
     # Emulate a column in a transposed dataframe (which is just a series)
     d = {
@@ -471,10 +505,10 @@ def test_country_not_matching_region(capsys):
     }
     series = pd.Series(data=d)
 
-    check.countries_match_regions(series)
+    check.countries_match_regions(series, exclude)
 
     captured = capsys.readouterr()
     assert (
         captured.out
-        == f"{Fore.YELLOW}Missing region ({missing_region}): {Fore.RESET}{title}\n"
+        == f"{Fore.YELLOW}Missing region ({country} → {missing_region}): {Fore.RESET}{title}\n"
     )

tests/test_fix.py

@@ -1,5 +1,7 @@
 # SPDX-License-Identifier: GPL-3.0-only
 
+import pandas as pd
+
 import csv_metadata_quality.fix as fix
 
 
@@ -120,3 +122,41 @@ def test_fix_mojibake():
     field_name = "dcterms.isPartOf"
 
     assert fix.mojibake(field, field_name) == "CIAT Publicaçao"
+
+
+def test_fix_country_not_matching_region():
+    """Test an item with regions not matching its country list."""
+
+    title = "Testing an item with no matching region."
+    country = "Kenya"
+    region = ""
+    missing_region = "Eastern Africa"
+    exclude = []
+
+    # Emulate a column in a transposed dataframe (which is just a series)
+    d = {
+        "dc.title": title,
+        "cg.coverage.country": country,
+        "cg.coverage.region": region,
+    }
+    series = pd.Series(data=d)
+
+    result = fix.countries_match_regions(series, exclude)
+
+    # Emulate the correct series we are expecting
+    d_correct = {
+        "dc.title": title,
+        "cg.coverage.country": country,
+        "cg.coverage.region": missing_region,
+    }
+    series_correct = pd.Series(data=d_correct)
+
+    pd.testing.assert_series_equal(result, series_correct)
+
+
+def test_fix_normalize_dois():
+    """Test normalizing a DOI."""
+
+    value = "doi: 10.11648/j.jps.20140201.14"
+
+    assert fix.normalize_dois(value) == "https://doi.org/10.11648/j.jps.20140201.14"