poetry.lock: run poetry update

We don't explicitly depend on PyArrow. It should come as a pandas
extra. I installed it like this:

    $ poetry add pandas=="^2.0.2[feather,performance]"

See: https://pandas.pydata.org/docs/getting_started/install.html#other-data-sources

.drone.yml

@@ -1,3 +1,33 @@
+---
+kind: pipeline
+type: docker
+name: python311
+
+steps:
+- name: test
+  image: python:3.11-slim
+  commands:
+  - id
+  - python -V
+  - apt update && apt install -y gcc g++ libicu-dev pkg-config git
+  - pip install -r requirements-dev.txt
+  - pytest
+  - python setup.py install
+  # Basic test
+  - csv-metadata-quality -i data/test.csv -o /tmp/test.csv
+  # Basic test with unsafe fixes
+  - csv-metadata-quality -i data/test.csv -o /tmp/test.csv -u
+  # Geography test
+  - csv-metadata-quality -i data/test-geography.csv -o /tmp/test.csv
+  # Geography test with unsafe fixes
+  - csv-metadata-quality -i data/test-geography.csv -o /tmp/test.csv -u
+  # Test with experimental checks
+  - csv-metadata-quality -i data/test.csv -o /tmp/test.csv -e
+  # Test with AGROVOC validation
+  - csv-metadata-quality -i data/test.csv -o /tmp/test.csv --agrovoc-fields dcterms.subject
+  # Test with AGROVOC validation (and dropping invalid)
+  - csv-metadata-quality -i data/test.csv -o /tmp/test.csv --agrovoc-fields dcterms.subject -d
+
 ---
 kind: pipeline
 type: docker

.github/workflows/python-app.yml

@@ -16,10 +16,10 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
-    - name: Set up Python 3.10
+    - name: Set up Python 3.11
       uses: actions/setup-python@v4
       with:
-        python-version: '3.10'
+        python-version: '3.11'
         cache: 'pip'
     - name: Install dependencies
       run: |

CHANGELOG.md

@@ -4,6 +4,19 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## Unreleased
+### Fixed
+- Fixed regex so we don't run the invalid multi-value separator fix on
+`dcterms.bibliographicCitation` fields
+- Fixed regex so we run the comma space fix on `dcterms.bibliographicCitation`
+fields
+
+### Changed
+- Don't run newline fix on description fields
+
+### Updated
+- Python dependencies, including Pandas 2.0.0 and [Arrow-backed dtypes](https://datapythonista.me/blog/pandas-20-and-the-arrow-revolution-part-i)
+
 ## [0.6.1] - 2023-02-23
 ### Fixed
 - Missing region check should ignore subregion field, if it exists

MANIFEST.in

@@ -0,0 +1 @@
+include csv_metadata_quality/data/licenses.json

README.md

@@ -127,7 +127,6 @@ This currently uses the [Python langid](https://github.com/saffsd/langid.py) lib
 - Warn if an author is shorter than 3 characters?
 - Validate DOIs? Normalize to https://doi.org format? Or use just the DOI part: 10.1016/j.worlddev.2010.06.006
 - Warn if two items use the same file in `filename` column
-- Add an option to drop invalid AGROVOC subjects?
 - Add tests for application invocation, ie `tests/test_app.py`?
 - Validate ISSNs or journal titles against CrossRef API?
 - Add configurable field validation, like specify a field name and a validation file?
@@ -137,7 +136,7 @@ This currently uses the [Python langid](https://github.com/saffsd/langid.py) lib
   - Warn if item is Open Access, but missing a license
   - Warn if item has an ISSN but no journal title
   - Update journal titles from ISSN
-- Migrate to https://github.com/spdx/license-list-data
+- Migrate from Pandas to Polars
 
 ## License
 This work is licensed under the [GPLv3](https://www.gnu.org/licenses/gpl-3.0.en.html).

csv_metadata_quality/app.py

@@ -73,8 +73,7 @@ def run(argv):
     # set the signal handler for SIGINT (^C)
     signal.signal(signal.SIGINT, signal_handler)
 
-    # Read all fields as strings so dates don't get converted from 1998 to 1998.0
+    df = pd.read_csv(args.input_file, dtype_backend="pyarrow")
-    df = pd.read_csv(args.input_file, dtype=str)
 
     # Check if the user requested to skip any fields
     if args.exclude_fields:
@@ -91,7 +90,9 @@ def run(argv):
             continue
 
         if args.unsafe_fixes:
-            match = re.match(r"^.*?abstract.*$", column)
+            # Skip whitespace and newline fixes on abstracts and descriptions
+            # because there are too many with legitimate multi-line metadata.
+            match = re.match(r"^.*?(abstract|description).*$", column)
             if match is None:
                 # Fix: whitespace
                 df[column] = df[column].apply(fix.whitespace, field_name=column)
@@ -102,7 +103,7 @@ def run(argv):
         # Fix: missing space after comma. Only run on author and citation
         # fields for now, as this problem is mostly an issue in names.
         if args.unsafe_fixes:
-            match = re.match(r"^.*?(author|citation).*$", column)
+            match = re.match(r"^.*?(author|[Cc]itation).*$", column)
             if match is not None:
                 df[column] = df[column].apply(fix.comma_space, field_name=column)
 
@@ -126,7 +127,7 @@ def run(argv):
         # Fix: invalid and unnecessary multi-value separators. Skip the title
         # and abstract fields because "|" is used to indicate something like
         # a subtitle.
-        match = re.match(r"^.*?(abstract|title).*$", column)
+        match = re.match(r"^.*?(abstract|[Cc]itation|title).*$", column)
         if match is None:
             df[column] = df[column].apply(fix.separators, field_name=column)
             # Run whitespace fix again after fixing invalid separators

csv_metadata_quality/util.py

@@ -2,7 +2,7 @@
 
 
 import json
-from importlib.resources import files
+import os
 
 from ftfy.badness import is_bad
 
@@ -58,7 +58,7 @@ def is_mojibake(field):
 def load_spdx_licenses():
     """Returns a Python list of SPDX short license identifiers."""
 
-    with open(files("csv_metadata_quality").joinpath("data/licenses.json")) as f:
+    with open(os.path.join(os.path.dirname(__file__), "data/licenses.json")) as f:
         licenses = json.load(f)
 
     # List comprehension to extract the license ID for each license

pyproject.toml

@@ -12,25 +12,23 @@ csv-metadata-quality = 'csv_metadata_quality.__main__:main'
 
 [tool.poetry.dependencies]
 python = "^3.9"
-pandas = "^1.5.2"
+pandas = {version = "^2.0.2", extras = ["feather", "performance"]}
 python-stdnum = "^1.18"
 requests = "^2.28.2"
 requests-cache = "^0.9.8"
 langid = "^1.1.6"
 colorama = "^0.4.6"
 ftfy = "^6.1.1"
-country-converter = {git = "https://github.com/alanorth/country_converter.git", rev = "myanmar-region"}
+country-converter = "~1.0.0"
-pycountry = {git = "https://github.com/alanorth/pycountry", rev = "iso-codes-4.12.0"}
+pycountry = {git = "https://github.com/alanorth/pycountry", rev = "iso-codes-4.13.0"}
 
-[tool.poetry.dev-dependencies]
+[tool.poetry.group.dev.dependencies]
 pytest = "^7.2.1"
 flake8 = "^6.0.0"
 pytest-clarity = "^1.0.1"
 black = "^23.1.0"
 isort = "^5.12.0"
 csvkit = "^1.1.0"
-
-[tool.poetry.group.dev.dependencies]
 ipython = "^8.10.0"
 
 [build-system]

renovate.json

@@ -0,0 +1,9 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:base"
+  ],
+  "pip_requirements": {
+      "enabled": false
+  }
+}

requirements-dev.txt

@@ -5,28 +5,28 @@ agate==1.7.1 ; python_version >= "3.9" and python_version < "4.0"
 appdirs==1.4.4 ; python_version >= "3.9" and python_version < "4.0"
 appnope==0.1.3 ; python_version >= "3.9" and python_version < "4.0" and sys_platform == "darwin"
 asttokens==2.2.1 ; python_version >= "3.9" and python_version < "4.0"
-attrs==22.2.0 ; python_version >= "3.9" and python_version < "4.0"
+attrs==23.1.0 ; python_version >= "3.9" and python_version < "4.0"
-babel==2.11.0 ; python_version >= "3.9" and python_version < "4.0"
+babel==2.12.1 ; python_version >= "3.9" and python_version < "4.0"
 backcall==0.2.0 ; python_version >= "3.9" and python_version < "4.0"
-black==23.1.0 ; python_version >= "3.9" and python_version < "4.0"
+black==23.3.0 ; python_version >= "3.9" and python_version < "4.0"
 cattrs==22.2.0 ; python_version >= "3.9" and python_version < "4.0"
-certifi==2022.12.7 ; python_version >= "3.9" and python_version < "4"
+certifi==2022.12.7 ; python_version >= "3.9" and python_version < "4.0"
-charset-normalizer==3.0.1 ; python_version >= "3.9" and python_version < "4"
+charset-normalizer==3.1.0 ; python_version >= "3.9" and python_version < "4.0"
 click==8.1.3 ; python_version >= "3.9" and python_version < "4.0"
 colorama==0.4.6 ; python_version >= "3.9" and python_version < "4.0"
-country-converter @ git+https://github.com/alanorth/country_converter.git@myanmar-region ; python_version >= "3.9" and python_version < "4.0"
+country-converter==1.0.0 ; python_version >= "3.9" and python_version < "4.0"
 csvkit==1.1.1 ; python_version >= "3.9" and python_version < "4.0"
 dbfread==2.0.7 ; python_version >= "3.9" and python_version < "4.0"
 decorator==5.1.1 ; python_version >= "3.9" and python_version < "4.0"
 et-xmlfile==1.1.0 ; python_version >= "3.9" and python_version < "4.0"
-exceptiongroup==1.1.0 ; python_version >= "3.9" and python_version < "3.11"
+exceptiongroup==1.1.1 ; python_version >= "3.9" and python_version < "3.11"
 executing==1.2.0 ; python_version >= "3.9" and python_version < "4.0"
 flake8==6.0.0 ; python_version >= "3.9" and python_version < "4.0"
 ftfy==6.1.1 ; python_version >= "3.9" and python_version < "4"
-greenlet==2.0.2 ; python_version >= "3.9" and (platform_machine == "aarch64" or platform_machine == "ppc64le" or platform_machine == "x86_64" or platform_machine == "amd64" or platform_machine == "AMD64" or platform_machine == "win32" or platform_machine == "WIN32") and python_version < "4.0"
+greenlet==2.0.2 ; python_version >= "3.9" and platform_machine == "aarch64" and python_version < "4.0" or python_version >= "3.9" and platform_machine == "ppc64le" and python_version < "4.0" or python_version >= "3.9" and platform_machine == "x86_64" and python_version < "4.0" or python_version >= "3.9" and platform_machine == "amd64" and python_version < "4.0" or python_version >= "3.9" and platform_machine == "AMD64" and python_version < "4.0" or python_version >= "3.9" and platform_machine == "win32" and python_version < "4.0" or python_version >= "3.9" and platform_machine == "WIN32" and python_version < "4.0"
-idna==3.4 ; python_version >= "3.9" and python_version < "4"
+idna==3.4 ; python_version >= "3.9" and python_version < "4.0"
 iniconfig==2.0.0 ; python_version >= "3.9" and python_version < "4.0"
-ipython==8.10.0 ; python_version >= "3.9" and python_version < "4.0"
+ipython==8.13.1 ; python_version >= "3.9" and python_version < "4.0"
 isodate==0.6.1 ; python_version >= "3.9" and python_version < "4.0"
 isort==5.12.0 ; python_version >= "3.9" and python_version < "4.0"
 jedi==0.18.2 ; python_version >= "3.9" and python_version < "4.0"
@@ -37,44 +37,46 @@ matplotlib-inline==0.1.6 ; python_version >= "3.9" and python_version < "4.0"
 mccabe==0.7.0 ; python_version >= "3.9" and python_version < "4.0"
 mdurl==0.1.2 ; python_version >= "3.9" and python_version < "4.0"
 mypy-extensions==1.0.0 ; python_version >= "3.9" and python_version < "4.0"
-numpy==1.24.2 ; python_version < "4.0" and python_version >= "3.9"
+numpy==1.24.3 ; python_version >= "3.9" and python_version < "4.0"
 olefile==0.46 ; python_version >= "3.9" and python_version < "4.0"
-openpyxl==3.1.1 ; python_version >= "3.9" and python_version < "4.0"
+openpyxl==3.1.2 ; python_version >= "3.9" and python_version < "4.0"
-packaging==23.0 ; python_version >= "3.9" and python_version < "4.0"
+packaging==23.1 ; python_version >= "3.9" and python_version < "4.0"
-pandas==1.5.3 ; python_version >= "3.9" and python_version < "4.0"
+pandas==2.0.1 ; python_version >= "3.9" and python_version < "4.0"
 parsedatetime==2.6 ; python_version >= "3.9" and python_version < "4.0"
 parso==0.8.3 ; python_version >= "3.9" and python_version < "4.0"
-pathspec==0.11.0 ; python_version >= "3.9" and python_version < "4.0"
+pathspec==0.11.1 ; python_version >= "3.9" and python_version < "4.0"
 pexpect==4.8.0 ; python_version >= "3.9" and python_version < "4.0" and sys_platform != "win32"
 pickleshare==0.7.5 ; python_version >= "3.9" and python_version < "4.0"
-platformdirs==3.0.0 ; python_version >= "3.9" and python_version < "4.0"
+platformdirs==3.5.0 ; python_version >= "3.9" and python_version < "4.0"
 pluggy==1.0.0 ; python_version >= "3.9" and python_version < "4.0"
 pprintpp==0.4.0 ; python_version >= "3.9" and python_version < "4.0"
-prompt-toolkit==3.0.37 ; python_version >= "3.9" and python_version < "4.0"
+prompt-toolkit==3.0.38 ; python_version >= "3.9" and python_version < "4.0"
 ptyprocess==0.7.0 ; python_version >= "3.9" and python_version < "4.0" and sys_platform != "win32"
 pure-eval==0.2.2 ; python_version >= "3.9" and python_version < "4.0"
+pyarrow==11.0.0 ; python_version >= "3.9" and python_version < "4.0"
 pycodestyle==2.10.0 ; python_version >= "3.9" and python_version < "4.0"
-pycountry @ git+https://github.com/alanorth/pycountry@iso-codes-4.12.0 ; python_version >= "3.9" and python_version < "4.0"
+pycountry @ git+https://github.com/alanorth/pycountry@iso-codes-4.13.0 ; python_version >= "3.9" and python_version < "4.0"
 pyflakes==3.0.1 ; python_version >= "3.9" and python_version < "4.0"
-pygments==2.14.0 ; python_version >= "3.9" and python_version < "4.0"
+pygments==2.15.1 ; python_version >= "3.9" and python_version < "4.0"
 pytest-clarity==1.0.1 ; python_version >= "3.9" and python_version < "4.0"
-pytest==7.2.1 ; python_version >= "3.9" and python_version < "4.0"
+pytest==7.3.1 ; python_version >= "3.9" and python_version < "4.0"
 python-dateutil==2.8.2 ; python_version >= "3.9" and python_version < "4.0"
-python-slugify==8.0.0 ; python_version >= "3.9" and python_version < "4.0"
+python-slugify==8.0.1 ; python_version >= "3.9" and python_version < "4.0"
 python-stdnum==1.18 ; python_version >= "3.9" and python_version < "4.0"
 pytimeparse==1.1.8 ; python_version >= "3.9" and python_version < "4.0"
-pytz==2022.7.1 ; python_version >= "3.9" and python_version < "4.0"
+pytz==2023.3 ; python_version >= "3.9" and python_version < "4.0"
 requests-cache==0.9.8 ; python_version >= "3.9" and python_version < "4.0"
-requests==2.28.2 ; python_version >= "3.9" and python_version < "4"
+requests==2.29.0 ; python_version >= "3.9" and python_version < "4.0"
-rich==13.3.1 ; python_version >= "3.9" and python_version < "4.0"
+rich==13.3.5 ; python_version >= "3.9" and python_version < "4.0"
 six==1.16.0 ; python_version >= "3.9" and python_version < "4.0"
-sqlalchemy==1.4.46 ; python_version >= "3.9" and python_version < "4.0"
+sqlalchemy==1.4.48 ; python_version >= "3.9" and python_version < "4.0"
 stack-data==0.6.2 ; python_version >= "3.9" and python_version < "4.0"
 text-unidecode==1.3 ; python_version >= "3.9" and python_version < "4.0"
 tomli==2.0.1 ; python_version >= "3.9" and python_version < "3.11"
 traitlets==5.9.0 ; python_version >= "3.9" and python_version < "4.0"
 typing-extensions==4.5.0 ; python_version >= "3.9" and python_version < "3.10"
+tzdata==2023.3 ; python_version >= "3.9" and python_version < "4.0"
 url-normalize==1.4.3 ; python_version >= "3.9" and python_version < "4.0"
-urllib3==1.26.14 ; python_version >= "3.9" and python_version < "4"
+urllib3==1.26.15 ; python_version >= "3.9" and python_version < "4.0"
 wcwidth==0.2.6 ; python_version >= "3.9" and python_version < "4"
 xlrd==2.0.1 ; python_version >= "3.9" and python_version < "4.0"

requirements.txt

@@ -1,23 +1,25 @@
 appdirs==1.4.4 ; python_version >= "3.9" and python_version < "4.0"
-attrs==22.2.0 ; python_version >= "3.9" and python_version < "4.0"
+attrs==23.1.0 ; python_version >= "3.9" and python_version < "4.0"
 cattrs==22.2.0 ; python_version >= "3.9" and python_version < "4.0"
-certifi==2022.12.7 ; python_version >= "3.9" and python_version < "4"
+certifi==2022.12.7 ; python_version >= "3.9" and python_version < "4.0"
-charset-normalizer==3.0.1 ; python_version >= "3.9" and python_version < "4"
+charset-normalizer==3.1.0 ; python_version >= "3.9" and python_version < "4.0"
 colorama==0.4.6 ; python_version >= "3.9" and python_version < "4.0"
-country-converter @ git+https://github.com/alanorth/country_converter.git@myanmar-region ; python_version >= "3.9" and python_version < "4.0"
+country-converter==1.0.0 ; python_version >= "3.9" and python_version < "4.0"
-exceptiongroup==1.1.0 ; python_version >= "3.9" and python_version < "3.11"
+exceptiongroup==1.1.1 ; python_version >= "3.9" and python_version < "3.11"
 ftfy==6.1.1 ; python_version >= "3.9" and python_version < "4"
-idna==3.4 ; python_version >= "3.9" and python_version < "4"
+idna==3.4 ; python_version >= "3.9" and python_version < "4.0"
 langid==1.1.6 ; python_version >= "3.9" and python_version < "4.0"
-numpy==1.24.2 ; python_version < "4.0" and python_version >= "3.9"
+numpy==1.24.3 ; python_version >= "3.9" and python_version < "4.0"
-pandas==1.5.3 ; python_version >= "3.9" and python_version < "4.0"
+pandas==2.0.1 ; python_version >= "3.9" and python_version < "4.0"
-pycountry @ git+https://github.com/alanorth/pycountry@iso-codes-4.12.0 ; python_version >= "3.9" and python_version < "4.0"
+pyarrow==11.0.0 ; python_version >= "3.9" and python_version < "4.0"
+pycountry @ git+https://github.com/alanorth/pycountry@iso-codes-4.13.0 ; python_version >= "3.9" and python_version < "4.0"
 python-dateutil==2.8.2 ; python_version >= "3.9" and python_version < "4.0"
 python-stdnum==1.18 ; python_version >= "3.9" and python_version < "4.0"
-pytz==2022.7.1 ; python_version >= "3.9" and python_version < "4.0"
+pytz==2023.3 ; python_version >= "3.9" and python_version < "4.0"
 requests-cache==0.9.8 ; python_version >= "3.9" and python_version < "4.0"
-requests==2.28.2 ; python_version >= "3.9" and python_version < "4"
+requests==2.29.0 ; python_version >= "3.9" and python_version < "4.0"
 six==1.16.0 ; python_version >= "3.9" and python_version < "4.0"
+tzdata==2023.3 ; python_version >= "3.9" and python_version < "4.0"
 url-normalize==1.4.3 ; python_version >= "3.9" and python_version < "4.0"
-urllib3==1.26.14 ; python_version >= "3.9" and python_version < "4"
+urllib3==1.26.15 ; python_version >= "3.9" and python_version < "4.0"
 wcwidth==0.2.6 ; python_version >= "3.9" and python_version < "4"

setup.py

@@ -25,6 +25,7 @@ setuptools.setup(
     classifiers=[
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
         "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
         "Operating System :: OS Independent",
     ],
@@ -32,5 +33,6 @@ setuptools.setup(
     entry_points={
         "console_scripts": ["csv-metadata-quality = csv_metadata_quality.__main__:main"]
     },
+    include_package_data=True,
     install_requires=install_requires,
 )