Merge pull request #24 from ilri/renovate/actions-checkout-4.x

Update actions/checkout action to v4

.github/workflows/python-app.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-22.04
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Install poetry
       run: pipx install poetry
     - uses: actions/setup-python@v4

CHANGELOG.md

@@ -14,6 +14,7 @@ fields
 
 ### Changed
 - Don't run newline fix on description fields
+- Install requests-cache in main run() function instead of check.agrovoc() function so we only incur the overhead once
 
 ### Updated
 - Python dependencies, including Pandas 2.0.0 and [Arrow-backed dtypes](https://datapythonista.me/blog/pandas-20-and-the-arrow-revolution-part-i)

csv_metadata_quality/app.py

@@ -1,11 +1,14 @@
 # SPDX-License-Identifier: GPL-3.0-only
 
 import argparse
+import os
 import re
 import signal
 import sys
+from datetime import timedelta
 
 import pandas as pd
+import requests_cache
 from colorama import Fore
 
 import csv_metadata_quality.check as check
@@ -84,6 +87,19 @@ def run(argv):
     else:
         exclude = list()
 
+    # enable transparent request cache with thirty days expiry
+    expire_after = timedelta(days=30)
+    # Allow overriding the location of the requests cache, just in case we are
+    # running in an environment where we can't write to the current working di-
+    # rectory (for example from csv-metadata-quality-web).
+    REQUESTS_CACHE_DIR = os.environ.get("REQUESTS_CACHE_DIR", ".")
+    requests_cache.install_cache(
+        f"{REQUESTS_CACHE_DIR}/agrovoc-response-cache", expire_after=expire_after
+    )
+
+    # prune old cache entries
+    requests_cache.delete()
+
     for column in df.columns:
         if column in exclude:
             print(f"{Fore.YELLOW}Skipping {Fore.RESET}{column}")

csv_metadata_quality/check.py

@@ -1,14 +1,12 @@
 # SPDX-License-Identifier: GPL-3.0-only
 
 import logging
-import os
 import re
 from datetime import datetime, timedelta
 
 import country_converter as coco
 import pandas as pd
 import requests
-import requests_cache
 from colorama import Fore
 from pycountry import languages
 from stdnum import isbn as stdnum_isbn
@@ -203,25 +201,12 @@ def agrovoc(field, field_name, drop):
     if pd.isna(field):
         return
 
-    # enable transparent request cache with thirty days expiry
-    expire_after = timedelta(days=30)
-    # Allow overriding the location of the requests cache, just in case we are
-    # running in an environment where we can't write to the current working di-
-    # rectory (for example from csv-metadata-quality-web).
-    REQUESTS_CACHE_DIR = os.environ.get("REQUESTS_CACHE_DIR", ".")
-    requests_cache.install_cache(
-        f"{REQUESTS_CACHE_DIR}/agrovoc-response-cache", expire_after=expire_after
-    )
-
-    # prune old cache entries
-    # requests_cache.remove_expired_responses()
-
     # Initialize an empty list to hold the validated AGROVOC values
     values = list()
 
     # Try to split multi-value field on "||" separator
     for value in field.split("||"):
-        request_url = "http://agrovoc.uniroma2.it/agrovoc/rest/v1/agrovoc/search"
+        request_url = "https://agrovoc.uniroma2.it/agrovoc/rest/v1/agrovoc/search"
         request_params = {"query": value}
 
         request = requests.get(request_url, params=request_params)

pyproject.toml

@@ -20,7 +20,7 @@ langid = "^1.1.6"
 colorama = "^0.4.6"
 ftfy = "^6.1.1"
 country-converter = "~1.0.0"
-pycountry = {git = "https://github.com/alanorth/pycountry", rev = "iso-codes-4.13.0"}
+pycountry = {git = "https://github.com/alanorth/pycountry", rev = "iso-codes-4.15.0"}
 
 [tool.poetry.group.dev.dependencies]
 pytest = "^7.2.1"