alanorth/csv-metadata-quality
1
0
Fork 0
mirror of https://github.com/ilri/csv-metadata-quality.git synced 2024-11-25 23:28:18 +01:00
Code Releases Activity

csv_metadata_quality/check.py: Prune requests cache once

Browse Source 
We only need to prune the requests cache once before using it, not
for every value we check.
This commit is contained in:
Alan Orth 2020-07-06 13:41:51 +03:00
parent aa9e23b46c
commit 5fcaa63bd5
Signed by: alanorth
GPG Key ID: 0FB860CC9C45B1B9
1 changed files with 9 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
csv_metadata_quality/check.py
View File

@ -221,23 +221,23 @@ def agrovoc(field, field_name):
if pd.isna(field): if pd.isna(field):
return return
# Try to split multi-value field on "||" separator
for value in field.split("||"):
request_url = (
f"http://agrovoc.uniroma2.it/agrovoc/rest/v1/agrovoc/search?query={value}"
)
# enable transparent request cache with thirty days expiry # enable transparent request cache with thirty days expiry
expire_after = timedelta(days=30) expire_after = timedelta(days=30)
requests_cache.install_cache( requests_cache.install_cache(
"agrovoc-response-cache", expire_after=expire_after "agrovoc-response-cache", expire_after=expire_after
) )
request = requests.get(request_url)
# prune old cache entries # prune old cache entries
requests_cache.core.remove_expired_responses() requests_cache.core.remove_expired_responses()
# Try to split multi-value field on "||" separator
for value in field.split("||"):
request_url = (
f"http://agrovoc.uniroma2.it/agrovoc/rest/v1/agrovoc/search?query={value}"
)
request = requests.get(request_url)
if request.status_code == requests.codes.ok: if request.status_code == requests.codes.ok:
data = request.json() data = request.json()