.github/workflows/python-app.yml

@@ -16,26 +16,30 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
-    - name: Install rye
+    - name: Install poetry
-      uses: eifinger/setup-rye@v4
+      run: pipx install poetry
+    - uses: actions/setup-python@v5
       with:
-        version: 'latest'
+        python-version: '3.11'
-    - run: rye sync
+        cache: 'poetry'
-    - name: Lint
+    - run: poetry install
+    - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names
-        rye lint
+        poetry run flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        poetry run flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
     - name: Test with pytest
-      run: rye test
+      run: poetry run pytest
     - name: Test CLI
       run: |
         # Basic test
-        rye run csv-metadata-quality -i data/test.csv -o /tmp/test.csv
+        poetry run csv-metadata-quality -i data/test.csv -o /tmp/test.csv
         # Test with unsafe fixes
-        rye run csv-metadata-quality -i data/test.csv -o /tmp/test.csv -u
+        poetry run csv-metadata-quality -i data/test.csv -o /tmp/test.csv -u
         # Test with experimental checks
-        rye run csv-metadata-quality -i data/test.csv -o /tmp/test.csv -e
+        poetry run csv-metadata-quality -i data/test.csv -o /tmp/test.csv -e
         # Test with AGROVOC validation
-        rye run csv-metadata-quality -i data/test.csv -o /tmp/test.csv --agrovoc-fields dcterms.subject
+        poetry run csv-metadata-quality -i data/test.csv -o /tmp/test.csv --agrovoc-fields dcterms.subject
         # Test with AGROVOC validation (and dropping invalid)
-        rye run csv-metadata-quality -i data/test.csv -o /tmp/test.csv --agrovoc-fields dcterms.subject -d
+        poetry run csv-metadata-quality -i data/test.csv -o /tmp/test.csv --agrovoc-fields dcterms.subject -d

.python-version

@@ -1 +0,0 @@
-3.12

csv_metadata_quality/fix.py

@@ -444,20 +444,6 @@ def normalize_dois(field):
         if match:
             new_value = re.sub(pattern, "doi.org", new_value)
 
-        # Convert www.doi.org to doi.org
-        pattern = re.compile(r"www\.doi\.org")
-        match = re.findall(pattern, new_value)
-
-        if match:
-            new_value = re.sub(pattern, "doi.org", new_value)
-
-        # Convert erroneous %2f to /
-        pattern = re.compile("%2f")
-        match = re.findall(pattern, new_value)
-
-        if match:
-            new_value = re.sub(pattern, "/", new_value)
-
         # Replace values like doi: 10.11648/j.jps.20140201.14
         pattern = re.compile(r"^doi: 10\.")
         match = re.findall(pattern, new_value)

data/test.csv

@@ -40,4 +40,3 @@ Subregion field shouldn’t trigger region checks,2022-12-07,,,,,Kenya,,,,,,East
 DOI with HTTP and dx.doi.org,2024-04-23,,,,,,,,,,http://dx.doi.org/10.1016/j.envc.2023.100794,,
 DOI with colon,2024-04-23,,,,,,,,,,doi: 10.11648/j.jps.20140201.14,,
 Upper case bare DOI,2024-04-23,,,,,,,,,,10.19103/AS.2018.0043.16,,
-DOI with %2f,2024-06-25,,,,,,,,,,https://doi.org/10.1016%2fj.envc.2023.100794,,

pyproject.toml

@@ -1,63 +1,40 @@
-[project]
+[tool.poetry]
 name = "csv-metadata-quality"
 version = "0.6.1"
 description="A simple, but opinionated CSV quality checking and fixing pipeline for CSVs in the DSpace ecosystem."
-authors = [
+authors = ["Alan Orth <alan.orth@gmail.com>"]
-    { name = "Alan Orth", email = "alan.orth@gmail.com" }
+license="GPL-3.0-only"
-]
-license= { file = "LICENSE.txt" }
-dependencies = [
-    "pandas[feather,performance]~=2.2",
-    "python-stdnum~=1.20",
-    "requests~=2.32",
-    "requests-cache~=1.2.1",
-    "colorama~=0.4",
-    "ftfy~=6.2.0",
-    "country-converter~=1.2",
-    "pycountry~=24.6.1",
-    "py3langid~=0.3",
-]
-readme = "README.md"
-requires-python = ">= 3.9"
-
-classifiers = [
-  "License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
-  "Natural Language :: English",
-  "Operating System :: OS Independent",
-  "Programming Language :: Python :: 3.9",
-  "Programming Language :: Python :: 3.10",
-  "Programming Language :: Python :: 3.11",
-  "Programming Language :: Python :: 3.12",
-  "Programming Language :: Python :: Implementation :: CPython",
-]
-
-[project.urls]
 repository = "https://github.com/ilri/csv-metadata-quality"
 homepage = "https://github.com/ilri/csv-metadata-quality"
 
-[project.scripts]
+[tool.poetry.scripts]
 csv-metadata-quality = 'csv_metadata_quality.__main__:main'
 
-# So rye doesn't fall back to setuptools
+[tool.poetry.dependencies]
-# See: https://packaging.python.org/en/latest/tutorials/packaging-projects/#choosing-build-backend
+python = "^3.9"
+pandas = {version = "^2.0.2", extras = ["feather", "performance"]}
+python-stdnum = "^1.18"
+requests = "^2.28.2"
+requests-cache = "^1.0.0"
+colorama = "^0.4.6"
+ftfy = "^6.1.1"
+country-converter = "~1.1.0"
+pycountry = "^23.12.7"
+py3langid = "^0.2.2"
+
+[tool.poetry.group.dev.dependencies]
+pytest = "^7.2.1"
+flake8 = "^7.0.0"
+pytest-clarity = "^1.0.1"
+black = "^23.1.0"
+isort = "^5.12.0"
+csvkit = "^1.1.0"
+ipython = "^8.10.0"
+fixit = "^2.1.0"
+
 [build-system]
-requires = ["hatchling"]
+requires = ["poetry>=0.12"]
-build-backend = "hatchling.build"
+build-backend = "poetry.masonry.api"
-
-[tool.rye]
-managed = true
-dev-dependencies = [
-    "pytest~=8.3",
-    "pytest-clarity~=1.0",
-    "isort~=5.13",
-    "csvkit~=2.0",
-    "ipython~=8.26",
-    "fixit~=2.1",
-]
-
-# So hatch doesn't try to build other top-level directories like "data"
-[tool.hatch.build.targets.wheel]
-packages = ["csv_metadata_quality"]
 
 [tool.isort]
 profile = "black"

requirements-dev.lock

@@ -1,188 +0,0 @@
-# generated by rye
-# use `rye lock` or `rye sync` to update this lockfile
-#
-# last locked with the following flags:
-#   pre: false
-#   features: []
-#   all-features: false
-#   with-sources: false
-#   generate-hashes: false
-#   universal: false
-
--e file:.
-agate==1.10.2
-    # via agate-dbf
-    # via agate-excel
-    # via agate-sql
-    # via csvkit
-agate-dbf==0.2.3
-    # via csvkit
-agate-excel==0.4.1
-    # via csvkit
-agate-sql==0.7.2
-    # via csvkit
-asttokens==2.4.1
-    # via stack-data
-attrs==23.2.0
-    # via cattrs
-    # via requests-cache
-babel==2.15.0
-    # via agate
-bottleneck==1.3.8
-    # via pandas
-cattrs==23.2.3
-    # via requests-cache
-certifi==2024.2.2
-    # via requests
-charset-normalizer==3.3.2
-    # via requests
-click==8.1.7
-    # via fixit
-    # via moreorless
-colorama==0.4.6
-    # via csv-metadata-quality
-country-converter==1.2
-    # via csv-metadata-quality
-csvkit==2.0.1
-dbfread==2.0.7
-    # via agate-dbf
-decorator==5.1.1
-    # via ipython
-et-xmlfile==1.1.0
-    # via openpyxl
-executing==2.0.1
-    # via stack-data
-fixit==2.1.0
-ftfy==6.2.0
-    # via csv-metadata-quality
-greenlet==3.0.3
-    # via sqlalchemy
-idna==3.7
-    # via requests
-iniconfig==2.0.0
-    # via pytest
-ipython==8.26.0
-isodate==0.6.1
-    # via agate
-isort==5.13.2
-jedi==0.19.1
-    # via ipython
-leather==0.4.0
-    # via agate
-libcst==1.4.0
-    # via fixit
-llvmlite==0.43.0
-    # via numba
-markdown-it-py==3.0.0
-    # via rich
-matplotlib-inline==0.1.7
-    # via ipython
-mdurl==0.1.2
-    # via markdown-it-py
-moreorless==0.4.0
-    # via fixit
-numba==0.60.0
-    # via pandas
-numexpr==2.10.0
-    # via pandas
-numpy==2.0.0
-    # via bottleneck
-    # via numba
-    # via numexpr
-    # via pandas
-    # via py3langid
-    # via pyarrow
-olefile==0.47
-    # via agate-excel
-openpyxl==3.1.2
-    # via agate-excel
-    # via csvkit
-packaging==24.0
-    # via fixit
-    # via pytest
-pandas==2.2.2
-    # via country-converter
-    # via csv-metadata-quality
-parsedatetime==2.6
-    # via agate
-parso==0.8.4
-    # via jedi
-pathspec==0.12.1
-    # via trailrunner
-pexpect==4.9.0
-    # via ipython
-platformdirs==4.2.2
-    # via requests-cache
-pluggy==1.5.0
-    # via pytest
-pprintpp==0.4.0
-    # via pytest-clarity
-prompt-toolkit==3.0.43
-    # via ipython
-ptyprocess==0.7.0
-    # via pexpect
-pure-eval==0.2.2
-    # via stack-data
-py3langid==0.3.0
-    # via csv-metadata-quality
-pyarrow==16.1.0
-    # via pandas
-pycountry==24.6.1
-    # via csv-metadata-quality
-pygments==2.18.0
-    # via ipython
-    # via rich
-pytest==8.3.2
-    # via pytest-clarity
-pytest-clarity==1.0.1
-python-dateutil==2.9.0.post0
-    # via pandas
-python-slugify==8.0.4
-    # via agate
-python-stdnum==1.20
-    # via csv-metadata-quality
-pytimeparse==1.1.8
-    # via agate
-pytz==2024.1
-    # via pandas
-pyyaml==6.0.1
-    # via libcst
-requests==2.32.2
-    # via csv-metadata-quality
-    # via requests-cache
-requests-cache==1.2.1
-    # via csv-metadata-quality
-rich==13.7.1
-    # via pytest-clarity
-six==1.16.0
-    # via asttokens
-    # via isodate
-    # via python-dateutil
-    # via url-normalize
-sqlalchemy==2.0.30
-    # via agate-sql
-    # via csvkit
-stack-data==0.6.3
-    # via ipython
-text-unidecode==1.3
-    # via python-slugify
-trailrunner==1.4.0
-    # via fixit
-traitlets==5.14.3
-    # via ipython
-    # via matplotlib-inline
-typing-extensions==4.11.0
-    # via sqlalchemy
-tzdata==2024.1
-    # via pandas
-url-normalize==1.4.3
-    # via requests-cache
-urllib3==2.2.1
-    # via requests
-    # via requests-cache
-wcwidth==0.2.13
-    # via ftfy
-    # via prompt-toolkit
-xlrd==2.0.1
-    # via agate-excel
-    # via csvkit

requirements-dev.txt

@@ -0,0 +1,82 @@
+agate-dbf==0.2.2 ; python_version >= "3.9" and python_version < "4.0"
+agate-excel==0.2.5 ; python_version >= "3.9" and python_version < "4.0"
+agate-sql==0.5.9 ; python_version >= "3.9" and python_version < "4.0"
+agate==1.7.1 ; python_version >= "3.9" and python_version < "4.0"
+appdirs==1.4.4 ; python_version >= "3.9" and python_version < "4.0"
+appnope==0.1.3 ; python_version >= "3.9" and python_version < "4.0" and sys_platform == "darwin"
+asttokens==2.2.1 ; python_version >= "3.9" and python_version < "4.0"
+attrs==23.1.0 ; python_version >= "3.9" and python_version < "4.0"
+babel==2.12.1 ; python_version >= "3.9" and python_version < "4.0"
+backcall==0.2.0 ; python_version >= "3.9" and python_version < "4.0"
+black==23.3.0 ; python_version >= "3.9" and python_version < "4.0"
+cattrs==22.2.0 ; python_version >= "3.9" and python_version < "4.0"
+certifi==2022.12.7 ; python_version >= "3.9" and python_version < "4.0"
+charset-normalizer==3.1.0 ; python_version >= "3.9" and python_version < "4.0"
+click==8.1.3 ; python_version >= "3.9" and python_version < "4.0"
+colorama==0.4.6 ; python_version >= "3.9" and python_version < "4.0"
+country-converter==1.0.0 ; python_version >= "3.9" and python_version < "4.0"
+csvkit==1.1.1 ; python_version >= "3.9" and python_version < "4.0"
+dbfread==2.0.7 ; python_version >= "3.9" and python_version < "4.0"
+decorator==5.1.1 ; python_version >= "3.9" and python_version < "4.0"
+et-xmlfile==1.1.0 ; python_version >= "3.9" and python_version < "4.0"
+exceptiongroup==1.1.1 ; python_version >= "3.9" and python_version < "3.11"
+executing==1.2.0 ; python_version >= "3.9" and python_version < "4.0"
+flake8==6.0.0 ; python_version >= "3.9" and python_version < "4.0"
+ftfy==6.1.1 ; python_version >= "3.9" and python_version < "4"
+greenlet==2.0.2 ; python_version >= "3.9" and platform_machine == "aarch64" and python_version < "4.0" or python_version >= "3.9" and platform_machine == "ppc64le" and python_version < "4.0" or python_version >= "3.9" and platform_machine == "x86_64" and python_version < "4.0" or python_version >= "3.9" and platform_machine == "amd64" and python_version < "4.0" or python_version >= "3.9" and platform_machine == "AMD64" and python_version < "4.0" or python_version >= "3.9" and platform_machine == "win32" and python_version < "4.0" or python_version >= "3.9" and platform_machine == "WIN32" and python_version < "4.0"
+idna==3.4 ; python_version >= "3.9" and python_version < "4.0"
+iniconfig==2.0.0 ; python_version >= "3.9" and python_version < "4.0"
+ipython==8.13.1 ; python_version >= "3.9" and python_version < "4.0"
+isodate==0.6.1 ; python_version >= "3.9" and python_version < "4.0"
+isort==5.12.0 ; python_version >= "3.9" and python_version < "4.0"
+jedi==0.18.2 ; python_version >= "3.9" and python_version < "4.0"
+langid==1.1.6 ; python_version >= "3.9" and python_version < "4.0"
+leather==0.3.4 ; python_version >= "3.9" and python_version < "4.0"
+markdown-it-py==2.2.0 ; python_version >= "3.9" and python_version < "4.0"
+matplotlib-inline==0.1.6 ; python_version >= "3.9" and python_version < "4.0"
+mccabe==0.7.0 ; python_version >= "3.9" and python_version < "4.0"
+mdurl==0.1.2 ; python_version >= "3.9" and python_version < "4.0"
+mypy-extensions==1.0.0 ; python_version >= "3.9" and python_version < "4.0"
+numpy==1.24.3 ; python_version >= "3.9" and python_version < "4.0"
+olefile==0.46 ; python_version >= "3.9" and python_version < "4.0"
+openpyxl==3.1.2 ; python_version >= "3.9" and python_version < "4.0"
+packaging==23.1 ; python_version >= "3.9" and python_version < "4.0"
+pandas==2.0.1 ; python_version >= "3.9" and python_version < "4.0"
+parsedatetime==2.6 ; python_version >= "3.9" and python_version < "4.0"
+parso==0.8.3 ; python_version >= "3.9" and python_version < "4.0"
+pathspec==0.11.1 ; python_version >= "3.9" and python_version < "4.0"
+pexpect==4.8.0 ; python_version >= "3.9" and python_version < "4.0" and sys_platform != "win32"
+pickleshare==0.7.5 ; python_version >= "3.9" and python_version < "4.0"
+platformdirs==3.5.0 ; python_version >= "3.9" and python_version < "4.0"
+pluggy==1.0.0 ; python_version >= "3.9" and python_version < "4.0"
+pprintpp==0.4.0 ; python_version >= "3.9" and python_version < "4.0"
+prompt-toolkit==3.0.38 ; python_version >= "3.9" and python_version < "4.0"
+ptyprocess==0.7.0 ; python_version >= "3.9" and python_version < "4.0" and sys_platform != "win32"
+pure-eval==0.2.2 ; python_version >= "3.9" and python_version < "4.0"
+pyarrow==11.0.0 ; python_version >= "3.9" and python_version < "4.0"
+pycodestyle==2.10.0 ; python_version >= "3.9" and python_version < "4.0"
+pycountry @ git+https://github.com/alanorth/pycountry@iso-codes-4.13.0 ; python_version >= "3.9" and python_version < "4.0"
+pyflakes==3.0.1 ; python_version >= "3.9" and python_version < "4.0"
+pygments==2.15.1 ; python_version >= "3.9" and python_version < "4.0"
+pytest-clarity==1.0.1 ; python_version >= "3.9" and python_version < "4.0"
+pytest==7.3.1 ; python_version >= "3.9" and python_version < "4.0"
+python-dateutil==2.8.2 ; python_version >= "3.9" and python_version < "4.0"
+python-slugify==8.0.1 ; python_version >= "3.9" and python_version < "4.0"
+python-stdnum==1.18 ; python_version >= "3.9" and python_version < "4.0"
+pytimeparse==1.1.8 ; python_version >= "3.9" and python_version < "4.0"
+pytz==2023.3 ; python_version >= "3.9" and python_version < "4.0"
+requests-cache==0.9.8 ; python_version >= "3.9" and python_version < "4.0"
+requests==2.29.0 ; python_version >= "3.9" and python_version < "4.0"
+rich==13.3.5 ; python_version >= "3.9" and python_version < "4.0"
+six==1.16.0 ; python_version >= "3.9" and python_version < "4.0"
+sqlalchemy==1.4.48 ; python_version >= "3.9" and python_version < "4.0"
+stack-data==0.6.2 ; python_version >= "3.9" and python_version < "4.0"
+text-unidecode==1.3 ; python_version >= "3.9" and python_version < "4.0"
+tomli==2.0.1 ; python_version >= "3.9" and python_version < "3.11"
+traitlets==5.9.0 ; python_version >= "3.9" and python_version < "4.0"
+typing-extensions==4.5.0 ; python_version >= "3.9" and python_version < "3.10"
+tzdata==2023.3 ; python_version >= "3.9" and python_version < "4.0"
+url-normalize==1.4.3 ; python_version >= "3.9" and python_version < "4.0"
+urllib3==1.26.15 ; python_version >= "3.9" and python_version < "4.0"
+wcwidth==0.2.6 ; python_version >= "3.9" and python_version < "4"
+xlrd==2.0.1 ; python_version >= "3.9" and python_version < "4.0"

requirements.lock

@@ -1,78 +0,0 @@
-# generated by rye
-# use `rye lock` or `rye sync` to update this lockfile
-#
-# last locked with the following flags:
-#   pre: false
-#   features: []
-#   all-features: false
-#   with-sources: false
-#   generate-hashes: false
-#   universal: false
-
--e file:.
-attrs==23.2.0
-    # via cattrs
-    # via requests-cache
-bottleneck==1.3.8
-    # via pandas
-cattrs==23.2.3
-    # via requests-cache
-certifi==2024.2.2
-    # via requests
-charset-normalizer==3.3.2
-    # via requests
-colorama==0.4.6
-    # via csv-metadata-quality
-country-converter==1.2
-    # via csv-metadata-quality
-ftfy==6.2.0
-    # via csv-metadata-quality
-idna==3.7
-    # via requests
-llvmlite==0.43.0
-    # via numba
-numba==0.60.0
-    # via pandas
-numexpr==2.10.0
-    # via pandas
-numpy==2.0.0
-    # via bottleneck
-    # via numba
-    # via numexpr
-    # via pandas
-    # via py3langid
-    # via pyarrow
-pandas==2.2.2
-    # via country-converter
-    # via csv-metadata-quality
-platformdirs==4.2.2
-    # via requests-cache
-py3langid==0.3.0
-    # via csv-metadata-quality
-pyarrow==16.1.0
-    # via pandas
-pycountry==24.6.1
-    # via csv-metadata-quality
-python-dateutil==2.9.0.post0
-    # via pandas
-python-stdnum==1.20
-    # via csv-metadata-quality
-pytz==2024.1
-    # via pandas
-requests==2.32.2
-    # via csv-metadata-quality
-    # via requests-cache
-requests-cache==1.2.1
-    # via csv-metadata-quality
-six==1.16.0
-    # via python-dateutil
-    # via url-normalize
-tzdata==2024.1
-    # via pandas
-url-normalize==1.4.3
-    # via requests-cache
-urllib3==2.2.1
-    # via requests
-    # via requests-cache
-wcwidth==0.2.13
-    # via ftfy

requirements.txt

@@ -0,0 +1,25 @@
+appdirs==1.4.4 ; python_version >= "3.9" and python_version < "4.0"
+attrs==23.1.0 ; python_version >= "3.9" and python_version < "4.0"
+cattrs==22.2.0 ; python_version >= "3.9" and python_version < "4.0"
+certifi==2022.12.7 ; python_version >= "3.9" and python_version < "4.0"
+charset-normalizer==3.1.0 ; python_version >= "3.9" and python_version < "4.0"
+colorama==0.4.6 ; python_version >= "3.9" and python_version < "4.0"
+country-converter==1.0.0 ; python_version >= "3.9" and python_version < "4.0"
+exceptiongroup==1.1.1 ; python_version >= "3.9" and python_version < "3.11"
+ftfy==6.1.1 ; python_version >= "3.9" and python_version < "4"
+idna==3.4 ; python_version >= "3.9" and python_version < "4.0"
+langid==1.1.6 ; python_version >= "3.9" and python_version < "4.0"
+numpy==1.24.3 ; python_version >= "3.9" and python_version < "4.0"
+pandas==2.0.1 ; python_version >= "3.9" and python_version < "4.0"
+pyarrow==11.0.0 ; python_version >= "3.9" and python_version < "4.0"
+pycountry @ git+https://github.com/alanorth/pycountry@iso-codes-4.13.0 ; python_version >= "3.9" and python_version < "4.0"
+python-dateutil==2.8.2 ; python_version >= "3.9" and python_version < "4.0"
+python-stdnum==1.18 ; python_version >= "3.9" and python_version < "4.0"
+pytz==2023.3 ; python_version >= "3.9" and python_version < "4.0"
+requests-cache==0.9.8 ; python_version >= "3.9" and python_version < "4.0"
+requests==2.29.0 ; python_version >= "3.9" and python_version < "4.0"
+six==1.16.0 ; python_version >= "3.9" and python_version < "4.0"
+tzdata==2023.3 ; python_version >= "3.9" and python_version < "4.0"
+url-normalize==1.4.3 ; python_version >= "3.9" and python_version < "4.0"
+urllib3==1.26.15 ; python_version >= "3.9" and python_version < "4.0"
+wcwidth==0.2.6 ; python_version >= "3.9" and python_version < "4"