.github/workflows: use rye in CI

Use rye instead of poetry in CI.

.github/workflows/python-app.yml

@@ -16,30 +16,26 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
-    - name: Install poetry
+    - name: Install rye
-      run: pipx install poetry
+      uses: eifinger/setup-rye@v4
-    - uses: actions/setup-python@v5
       with:
-        python-version: '3.11'
+        version: 'latest'
-        cache: 'poetry'
+    - run: rye sync
-    - run: poetry install
+    - name: Lint
-    - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names
-        poetry run flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        rye lint
-        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        poetry run flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
     - name: Test with pytest
-      run: poetry run pytest
+      run: rye test
     - name: Test CLI
       run: |
         # Basic test
-        poetry run csv-metadata-quality -i data/test.csv -o /tmp/test.csv
+        rye run csv-metadata-quality -i data/test.csv -o /tmp/test.csv
         # Test with unsafe fixes
-        poetry run csv-metadata-quality -i data/test.csv -o /tmp/test.csv -u
+        rye run csv-metadata-quality -i data/test.csv -o /tmp/test.csv -u
         # Test with experimental checks
-        poetry run csv-metadata-quality -i data/test.csv -o /tmp/test.csv -e
+        rye run csv-metadata-quality -i data/test.csv -o /tmp/test.csv -e
         # Test with AGROVOC validation
-        poetry run csv-metadata-quality -i data/test.csv -o /tmp/test.csv --agrovoc-fields dcterms.subject
+        rye run csv-metadata-quality -i data/test.csv -o /tmp/test.csv --agrovoc-fields dcterms.subject
         # Test with AGROVOC validation (and dropping invalid)
-        poetry run csv-metadata-quality -i data/test.csv -o /tmp/test.csv --agrovoc-fields dcterms.subject -d
+        rye run csv-metadata-quality -i data/test.csv -o /tmp/test.csv --agrovoc-fields dcterms.subject -d

.python-version

@@ -0,0 +1 @@
+3.12

csv_metadata_quality/fix.py

@@ -444,6 +444,20 @@ def normalize_dois(field):
         if match:
             new_value = re.sub(pattern, "doi.org", new_value)
 
+        # Convert www.doi.org to doi.org
+        pattern = re.compile(r"www\.doi\.org")
+        match = re.findall(pattern, new_value)
+
+        if match:
+            new_value = re.sub(pattern, "doi.org", new_value)
+
+        # Convert erroneous %2f to /
+        pattern = re.compile("%2f")
+        match = re.findall(pattern, new_value)
+
+        if match:
+            new_value = re.sub(pattern, "/", new_value)
+
         # Replace values like doi: 10.11648/j.jps.20140201.14
         pattern = re.compile(r"^doi: 10\.")
         match = re.findall(pattern, new_value)

data/test.csv

@@ -40,3 +40,4 @@ Subregion field shouldn’t trigger region checks,2022-12-07,,,,,Kenya,,,,,,East
 DOI with HTTP and dx.doi.org,2024-04-23,,,,,,,,,,http://dx.doi.org/10.1016/j.envc.2023.100794,,
 DOI with colon,2024-04-23,,,,,,,,,,doi: 10.11648/j.jps.20140201.14,,
 Upper case bare DOI,2024-04-23,,,,,,,,,,10.19103/AS.2018.0043.16,,
+DOI with %2f,2024-06-25,,,,,,,,,,https://doi.org/10.1016%2fj.envc.2023.100794,,

pyproject.toml

@@ -1,40 +1,63 @@
-[tool.poetry]
+[project]
 name = "csv-metadata-quality"
 version = "0.6.1"
 description="A simple, but opinionated CSV quality checking and fixing pipeline for CSVs in the DSpace ecosystem."
-authors = ["Alan Orth <alan.orth@gmail.com>"]
+authors = [
-license="GPL-3.0-only"
+    { name = "Alan Orth", email = "alan.orth@gmail.com" }
+]
+license= { file = "LICENSE.txt" }
+dependencies = [
+    "pandas[feather,performance]~=2.2",
+    "python-stdnum~=1.20",
+    "requests~=2.32",
+    "requests-cache~=1.2.1",
+    "colorama~=0.4",
+    "ftfy~=6.2.0",
+    "country-converter~=1.2",
+    "pycountry~=24.6.1",
+    "py3langid~=0.3",
+]
+readme = "README.md"
+requires-python = ">= 3.9"
+
+classifiers = [
+  "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
+  "Natural Language :: English",
+  "Operating System :: OS Independent",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: Implementation :: CPython",
+]
+
+[project.urls]
 repository = "https://github.com/ilri/csv-metadata-quality"
 homepage = "https://github.com/ilri/csv-metadata-quality"
 
-[tool.poetry.scripts]
+[project.scripts]
 csv-metadata-quality = 'csv_metadata_quality.__main__:main'
 
-[tool.poetry.dependencies]
+# So rye doesn't fall back to setuptools
-python = "^3.9"
+# See: https://packaging.python.org/en/latest/tutorials/packaging-projects/#choosing-build-backend
-pandas = {version = "^2.0.2", extras = ["feather", "performance"]}
-python-stdnum = "^1.18"
-requests = "^2.28.2"
-requests-cache = "^1.0.0"
-colorama = "^0.4.6"
-ftfy = "^6.1.1"
-country-converter = "~1.1.0"
-pycountry = "^23.12.7"
-py3langid = "^0.2.2"
-
-[tool.poetry.group.dev.dependencies]
-pytest = "^7.2.1"
-flake8 = "^7.0.0"
-pytest-clarity = "^1.0.1"
-black = "^23.1.0"
-isort = "^5.12.0"
-csvkit = "^1.1.0"
-ipython = "^8.10.0"
-fixit = "^2.1.0"
-
 [build-system]
-requires = ["poetry>=0.12"]
+requires = ["hatchling"]
-build-backend = "poetry.masonry.api"
+build-backend = "hatchling.build"
+
+[tool.rye]
+managed = true
+dev-dependencies = [
+    "pytest~=8.3",
+    "pytest-clarity~=1.0",
+    "isort~=5.13",
+    "csvkit~=2.0",
+    "ipython~=8.26",
+    "fixit~=2.1",
+]
+
+# So hatch doesn't try to build other top-level directories like "data"
+[tool.hatch.build.targets.wheel]
+packages = ["csv_metadata_quality"]
 
 [tool.isort]
 profile = "black"

requirements-dev.lock

@@ -0,0 +1,188 @@
+# generated by rye
+# use `rye lock` or `rye sync` to update this lockfile
+#
+# last locked with the following flags:
+#   pre: false
+#   features: []
+#   all-features: false
+#   with-sources: false
+#   generate-hashes: false
+#   universal: false
+
+-e file:.
+agate==1.10.2
+    # via agate-dbf
+    # via agate-excel
+    # via agate-sql
+    # via csvkit
+agate-dbf==0.2.3
+    # via csvkit
+agate-excel==0.4.1
+    # via csvkit
+agate-sql==0.7.2
+    # via csvkit
+asttokens==2.4.1
+    # via stack-data
+attrs==23.2.0
+    # via cattrs
+    # via requests-cache
+babel==2.15.0
+    # via agate
+bottleneck==1.3.8
+    # via pandas
+cattrs==23.2.3
+    # via requests-cache
+certifi==2024.2.2
+    # via requests
+charset-normalizer==3.3.2
+    # via requests
+click==8.1.7
+    # via fixit
+    # via moreorless
+colorama==0.4.6
+    # via csv-metadata-quality
+country-converter==1.2
+    # via csv-metadata-quality
+csvkit==2.0.1
+dbfread==2.0.7
+    # via agate-dbf
+decorator==5.1.1
+    # via ipython
+et-xmlfile==1.1.0
+    # via openpyxl
+executing==2.0.1
+    # via stack-data
+fixit==2.1.0
+ftfy==6.2.0
+    # via csv-metadata-quality
+greenlet==3.0.3
+    # via sqlalchemy
+idna==3.7
+    # via requests
+iniconfig==2.0.0
+    # via pytest
+ipython==8.26.0
+isodate==0.6.1
+    # via agate
+isort==5.13.2
+jedi==0.19.1
+    # via ipython
+leather==0.4.0
+    # via agate
+libcst==1.4.0
+    # via fixit
+llvmlite==0.43.0
+    # via numba
+markdown-it-py==3.0.0
+    # via rich
+matplotlib-inline==0.1.7
+    # via ipython
+mdurl==0.1.2
+    # via markdown-it-py
+moreorless==0.4.0
+    # via fixit
+numba==0.60.0
+    # via pandas
+numexpr==2.10.0
+    # via pandas
+numpy==2.0.0
+    # via bottleneck
+    # via numba
+    # via numexpr
+    # via pandas
+    # via py3langid
+    # via pyarrow
+olefile==0.47
+    # via agate-excel
+openpyxl==3.1.2
+    # via agate-excel
+    # via csvkit
+packaging==24.0
+    # via fixit
+    # via pytest
+pandas==2.2.2
+    # via country-converter
+    # via csv-metadata-quality
+parsedatetime==2.6
+    # via agate
+parso==0.8.4
+    # via jedi
+pathspec==0.12.1
+    # via trailrunner
+pexpect==4.9.0
+    # via ipython
+platformdirs==4.2.2
+    # via requests-cache
+pluggy==1.5.0
+    # via pytest
+pprintpp==0.4.0
+    # via pytest-clarity
+prompt-toolkit==3.0.43
+    # via ipython
+ptyprocess==0.7.0
+    # via pexpect
+pure-eval==0.2.2
+    # via stack-data
+py3langid==0.3.0
+    # via csv-metadata-quality
+pyarrow==16.1.0
+    # via pandas
+pycountry==24.6.1
+    # via csv-metadata-quality
+pygments==2.18.0
+    # via ipython
+    # via rich
+pytest==8.3.2
+    # via pytest-clarity
+pytest-clarity==1.0.1
+python-dateutil==2.9.0.post0
+    # via pandas
+python-slugify==8.0.4
+    # via agate
+python-stdnum==1.20
+    # via csv-metadata-quality
+pytimeparse==1.1.8
+    # via agate
+pytz==2024.1
+    # via pandas
+pyyaml==6.0.1
+    # via libcst
+requests==2.32.2
+    # via csv-metadata-quality
+    # via requests-cache
+requests-cache==1.2.1
+    # via csv-metadata-quality
+rich==13.7.1
+    # via pytest-clarity
+six==1.16.0
+    # via asttokens
+    # via isodate
+    # via python-dateutil
+    # via url-normalize
+sqlalchemy==2.0.30
+    # via agate-sql
+    # via csvkit
+stack-data==0.6.3
+    # via ipython
+text-unidecode==1.3
+    # via python-slugify
+trailrunner==1.4.0
+    # via fixit
+traitlets==5.14.3
+    # via ipython
+    # via matplotlib-inline
+typing-extensions==4.11.0
+    # via sqlalchemy
+tzdata==2024.1
+    # via pandas
+url-normalize==1.4.3
+    # via requests-cache
+urllib3==2.2.1
+    # via requests
+    # via requests-cache
+wcwidth==0.2.13
+    # via ftfy
+    # via prompt-toolkit
+xlrd==2.0.1
+    # via agate-excel
+    # via csvkit

requirements-dev.txt

@@ -1,82 +0,0 @@
-agate-dbf==0.2.2 ; python_version >= "3.9" and python_version < "4.0"
-agate-excel==0.2.5 ; python_version >= "3.9" and python_version < "4.0"
-agate-sql==0.5.9 ; python_version >= "3.9" and python_version < "4.0"
-agate==1.7.1 ; python_version >= "3.9" and python_version < "4.0"
-appdirs==1.4.4 ; python_version >= "3.9" and python_version < "4.0"
-appnope==0.1.3 ; python_version >= "3.9" and python_version < "4.0" and sys_platform == "darwin"
-asttokens==2.2.1 ; python_version >= "3.9" and python_version < "4.0"
-attrs==23.1.0 ; python_version >= "3.9" and python_version < "4.0"
-babel==2.12.1 ; python_version >= "3.9" and python_version < "4.0"
-backcall==0.2.0 ; python_version >= "3.9" and python_version < "4.0"
-black==23.3.0 ; python_version >= "3.9" and python_version < "4.0"
-cattrs==22.2.0 ; python_version >= "3.9" and python_version < "4.0"
-certifi==2022.12.7 ; python_version >= "3.9" and python_version < "4.0"
-charset-normalizer==3.1.0 ; python_version >= "3.9" and python_version < "4.0"
-click==8.1.3 ; python_version >= "3.9" and python_version < "4.0"
-colorama==0.4.6 ; python_version >= "3.9" and python_version < "4.0"
-country-converter==1.0.0 ; python_version >= "3.9" and python_version < "4.0"
-csvkit==1.1.1 ; python_version >= "3.9" and python_version < "4.0"
-dbfread==2.0.7 ; python_version >= "3.9" and python_version < "4.0"
-decorator==5.1.1 ; python_version >= "3.9" and python_version < "4.0"
-et-xmlfile==1.1.0 ; python_version >= "3.9" and python_version < "4.0"
-exceptiongroup==1.1.1 ; python_version >= "3.9" and python_version < "3.11"
-executing==1.2.0 ; python_version >= "3.9" and python_version < "4.0"
-flake8==6.0.0 ; python_version >= "3.9" and python_version < "4.0"
-ftfy==6.1.1 ; python_version >= "3.9" and python_version < "4"
-greenlet==2.0.2 ; python_version >= "3.9" and platform_machine == "aarch64" and python_version < "4.0" or python_version >= "3.9" and platform_machine == "ppc64le" and python_version < "4.0" or python_version >= "3.9" and platform_machine == "x86_64" and python_version < "4.0" or python_version >= "3.9" and platform_machine == "amd64" and python_version < "4.0" or python_version >= "3.9" and platform_machine == "AMD64" and python_version < "4.0" or python_version >= "3.9" and platform_machine == "win32" and python_version < "4.0" or python_version >= "3.9" and platform_machine == "WIN32" and python_version < "4.0"
-idna==3.4 ; python_version >= "3.9" and python_version < "4.0"
-iniconfig==2.0.0 ; python_version >= "3.9" and python_version < "4.0"
-ipython==8.13.1 ; python_version >= "3.9" and python_version < "4.0"
-isodate==0.6.1 ; python_version >= "3.9" and python_version < "4.0"
-isort==5.12.0 ; python_version >= "3.9" and python_version < "4.0"
-jedi==0.18.2 ; python_version >= "3.9" and python_version < "4.0"
-langid==1.1.6 ; python_version >= "3.9" and python_version < "4.0"
-leather==0.3.4 ; python_version >= "3.9" and python_version < "4.0"
-markdown-it-py==2.2.0 ; python_version >= "3.9" and python_version < "4.0"
-matplotlib-inline==0.1.6 ; python_version >= "3.9" and python_version < "4.0"
-mccabe==0.7.0 ; python_version >= "3.9" and python_version < "4.0"
-mdurl==0.1.2 ; python_version >= "3.9" and python_version < "4.0"
-mypy-extensions==1.0.0 ; python_version >= "3.9" and python_version < "4.0"
-numpy==1.24.3 ; python_version >= "3.9" and python_version < "4.0"
-olefile==0.46 ; python_version >= "3.9" and python_version < "4.0"
-openpyxl==3.1.2 ; python_version >= "3.9" and python_version < "4.0"
-packaging==23.1 ; python_version >= "3.9" and python_version < "4.0"
-pandas==2.0.1 ; python_version >= "3.9" and python_version < "4.0"
-parsedatetime==2.6 ; python_version >= "3.9" and python_version < "4.0"
-parso==0.8.3 ; python_version >= "3.9" and python_version < "4.0"
-pathspec==0.11.1 ; python_version >= "3.9" and python_version < "4.0"
-pexpect==4.8.0 ; python_version >= "3.9" and python_version < "4.0" and sys_platform != "win32"
-pickleshare==0.7.5 ; python_version >= "3.9" and python_version < "4.0"
-platformdirs==3.5.0 ; python_version >= "3.9" and python_version < "4.0"
-pluggy==1.0.0 ; python_version >= "3.9" and python_version < "4.0"
-pprintpp==0.4.0 ; python_version >= "3.9" and python_version < "4.0"
-prompt-toolkit==3.0.38 ; python_version >= "3.9" and python_version < "4.0"
-ptyprocess==0.7.0 ; python_version >= "3.9" and python_version < "4.0" and sys_platform != "win32"
-pure-eval==0.2.2 ; python_version >= "3.9" and python_version < "4.0"
-pyarrow==11.0.0 ; python_version >= "3.9" and python_version < "4.0"
-pycodestyle==2.10.0 ; python_version >= "3.9" and python_version < "4.0"
-pycountry @ git+https://github.com/alanorth/pycountry@iso-codes-4.13.0 ; python_version >= "3.9" and python_version < "4.0"
-pyflakes==3.0.1 ; python_version >= "3.9" and python_version < "4.0"
-pygments==2.15.1 ; python_version >= "3.9" and python_version < "4.0"
-pytest-clarity==1.0.1 ; python_version >= "3.9" and python_version < "4.0"
-pytest==7.3.1 ; python_version >= "3.9" and python_version < "4.0"
-python-dateutil==2.8.2 ; python_version >= "3.9" and python_version < "4.0"
-python-slugify==8.0.1 ; python_version >= "3.9" and python_version < "4.0"
-python-stdnum==1.18 ; python_version >= "3.9" and python_version < "4.0"
-pytimeparse==1.1.8 ; python_version >= "3.9" and python_version < "4.0"
-pytz==2023.3 ; python_version >= "3.9" and python_version < "4.0"
-requests-cache==0.9.8 ; python_version >= "3.9" and python_version < "4.0"
-requests==2.29.0 ; python_version >= "3.9" and python_version < "4.0"
-rich==13.3.5 ; python_version >= "3.9" and python_version < "4.0"
-six==1.16.0 ; python_version >= "3.9" and python_version < "4.0"
-sqlalchemy==1.4.48 ; python_version >= "3.9" and python_version < "4.0"
-stack-data==0.6.2 ; python_version >= "3.9" and python_version < "4.0"
-text-unidecode==1.3 ; python_version >= "3.9" and python_version < "4.0"
-tomli==2.0.1 ; python_version >= "3.9" and python_version < "3.11"
-traitlets==5.9.0 ; python_version >= "3.9" and python_version < "4.0"
-typing-extensions==4.5.0 ; python_version >= "3.9" and python_version < "3.10"
-tzdata==2023.3 ; python_version >= "3.9" and python_version < "4.0"
-url-normalize==1.4.3 ; python_version >= "3.9" and python_version < "4.0"
-urllib3==1.26.15 ; python_version >= "3.9" and python_version < "4.0"
-wcwidth==0.2.6 ; python_version >= "3.9" and python_version < "4"
-xlrd==2.0.1 ; python_version >= "3.9" and python_version < "4.0"

requirements.lock

@@ -0,0 +1,78 @@
+# generated by rye
+# use `rye lock` or `rye sync` to update this lockfile
+#
+# last locked with the following flags:
+#   pre: false
+#   features: []
+#   all-features: false
+#   with-sources: false
+#   generate-hashes: false
+#   universal: false
+
+-e file:.
+attrs==23.2.0
+    # via cattrs
+    # via requests-cache
+bottleneck==1.3.8
+    # via pandas
+cattrs==23.2.3
+    # via requests-cache
+certifi==2024.2.2
+    # via requests
+charset-normalizer==3.3.2
+    # via requests
+colorama==0.4.6
+    # via csv-metadata-quality
+country-converter==1.2
+    # via csv-metadata-quality
+ftfy==6.2.0
+    # via csv-metadata-quality
+idna==3.7
+    # via requests
+llvmlite==0.43.0
+    # via numba
+numba==0.60.0
+    # via pandas
+numexpr==2.10.0
+    # via pandas
+numpy==2.0.0
+    # via bottleneck
+    # via numba
+    # via numexpr
+    # via pandas
+    # via py3langid
+    # via pyarrow
+pandas==2.2.2
+    # via country-converter
+    # via csv-metadata-quality
+platformdirs==4.2.2
+    # via requests-cache
+py3langid==0.3.0
+    # via csv-metadata-quality
+pyarrow==16.1.0
+    # via pandas
+pycountry==24.6.1
+    # via csv-metadata-quality
+python-dateutil==2.9.0.post0
+    # via pandas
+python-stdnum==1.20
+    # via csv-metadata-quality
+pytz==2024.1
+    # via pandas
+requests==2.32.2
+    # via csv-metadata-quality
+    # via requests-cache
+requests-cache==1.2.1
+    # via csv-metadata-quality
+six==1.16.0
+    # via python-dateutil
+    # via url-normalize
+tzdata==2024.1
+    # via pandas
+url-normalize==1.4.3
+    # via requests-cache
+urllib3==2.2.1
+    # via requests
+    # via requests-cache
+wcwidth==0.2.13
+    # via ftfy

requirements.txt

@@ -1,25 +0,0 @@
-appdirs==1.4.4 ; python_version >= "3.9" and python_version < "4.0"
-attrs==23.1.0 ; python_version >= "3.9" and python_version < "4.0"
-cattrs==22.2.0 ; python_version >= "3.9" and python_version < "4.0"
-certifi==2022.12.7 ; python_version >= "3.9" and python_version < "4.0"
-charset-normalizer==3.1.0 ; python_version >= "3.9" and python_version < "4.0"
-colorama==0.4.6 ; python_version >= "3.9" and python_version < "4.0"
-country-converter==1.0.0 ; python_version >= "3.9" and python_version < "4.0"
-exceptiongroup==1.1.1 ; python_version >= "3.9" and python_version < "3.11"
-ftfy==6.1.1 ; python_version >= "3.9" and python_version < "4"
-idna==3.4 ; python_version >= "3.9" and python_version < "4.0"
-langid==1.1.6 ; python_version >= "3.9" and python_version < "4.0"
-numpy==1.24.3 ; python_version >= "3.9" and python_version < "4.0"
-pandas==2.0.1 ; python_version >= "3.9" and python_version < "4.0"
-pyarrow==11.0.0 ; python_version >= "3.9" and python_version < "4.0"
-pycountry @ git+https://github.com/alanorth/pycountry@iso-codes-4.13.0 ; python_version >= "3.9" and python_version < "4.0"
-python-dateutil==2.8.2 ; python_version >= "3.9" and python_version < "4.0"
-python-stdnum==1.18 ; python_version >= "3.9" and python_version < "4.0"
-pytz==2023.3 ; python_version >= "3.9" and python_version < "4.0"
-requests-cache==0.9.8 ; python_version >= "3.9" and python_version < "4.0"
-requests==2.29.0 ; python_version >= "3.9" and python_version < "4.0"
-six==1.16.0 ; python_version >= "3.9" and python_version < "4.0"
-tzdata==2023.3 ; python_version >= "3.9" and python_version < "4.0"
-url-normalize==1.4.3 ; python_version >= "3.9" and python_version < "4.0"
-urllib3==1.26.15 ; python_version >= "3.9" and python_version < "4.0"
-wcwidth==0.2.6 ; python_version >= "3.9" and python_version < "4"