mirror of
https://github.com/ilri/dspace-statistics-api.git
synced 2025-05-10 15:16:02 +02:00
Compare commits
76 Commits
v1.4.0-dev
...
v1.4.3
Author | SHA1 | Date | |
---|---|---|---|
1a1a14a25f
|
|||
c09fc789e8
|
|||
134a4f1595
|
|||
12ebd1aed5
|
|||
e5f3201b65
|
|||
c1ce4fe233
|
|||
b2eb1878a5
|
|||
a0213c1c97
|
|||
cd03ca2b36
|
|||
c48e6a79c7
|
|||
a2e1695ecc
|
|||
b683bf211c
|
|||
3ab48743d6
|
|||
88173eaae9
|
|||
f557d33f36
|
|||
ffc4ff4a5c
|
|||
7551b34632
|
|||
5e71ec10eb
|
|||
f80d360cf9
|
|||
e70b59ecfe
|
|||
4d0828b6c0
|
|||
dabc4c0259
|
|||
4fd8af07c3
|
|||
4c5326a176
|
|||
3b1ccafab4
|
|||
58b5ae82d3
|
|||
562aaeef7d
|
|||
5cdba6acb1
|
|||
dd0937179c
|
|||
f0c6c004db
|
|||
6843f0a8ac
|
|||
f5fcfcc05a
|
|||
e8ac74b6d1
|
|||
14fc14daee
|
|||
871aae537a
|
|||
2fada6c6ff
|
|||
ef0991e352
|
|||
4502d6053c
|
|||
a524068cf6
|
|||
964d5dff06
|
|||
a9252d1771
|
|||
a63687d516
|
|||
73dc3a292e
|
|||
1e742bad41
|
|||
164008981e
|
|||
dd1769b954
|
|||
b009820fb4
|
|||
9830295978
|
|||
c93a4d7455
|
|||
2f8e4f8a0a | |||
0650c5985e
|
|||
d814f1c4f0
|
|||
00f30591c4
|
|||
acfe87b91a
|
|||
bc6d84dda2 | |||
889fb2f74a
|
|||
c42cd7a818
|
|||
f8bba59d66
|
|||
b8cb752a29
|
|||
09496aa2b5
|
|||
ff5dc7506d
|
|||
80a11ead97
|
|||
a282c95933
|
|||
fd7cc36306
|
|||
a20ff09570
|
|||
fdc0e73088
|
|||
b15afc9f39
|
|||
2bc18ef719
|
|||
49751b53f0
|
|||
d1c177e146
|
|||
33dc210452
|
|||
282d5f644a
|
|||
05e0e8bdca
|
|||
2567bb8604
|
|||
4af3c656a3
|
|||
4f8cd1097b
|
21
.build.yml
21
.build.yml
@ -1,21 +0,0 @@
|
||||
image: archlinux
|
||||
packages:
|
||||
- python-poetry
|
||||
- postgresql
|
||||
sources:
|
||||
- https://git.sr.ht/~alanorth/dspace-statistics-api
|
||||
tasks:
|
||||
- setup: |
|
||||
id
|
||||
psql --version
|
||||
sudo su - postgres -c "initdb --locale en_US.UTF-8 -E UTF8 -D '/var/lib/postgres/data'"
|
||||
sudo systemctl start postgresql
|
||||
createuser -U postgres dspacestatistics
|
||||
psql -U postgres -c "ALTER USER dspacestatistics WITH PASSWORD 'dspacestatistics'"
|
||||
createdb -U postgres -O dspacestatistics --encoding=UNICODE dspacestatistics
|
||||
cd dspace-statistics-api
|
||||
psql -U postgres -d dspacestatistics < tests/dspacestatistics.sql
|
||||
poetry install --no-root
|
||||
- test: |
|
||||
cd dspace-statistics-api
|
||||
poetry run pytest
|
44
.drone.yml
44
.drone.yml
@ -1,5 +1,44 @@
|
||||
kind: pipeline
|
||||
type: docker
|
||||
name: python310
|
||||
|
||||
steps:
|
||||
- name: setup
|
||||
image: postgres:10-alpine
|
||||
environment:
|
||||
PGPASSWORD: postgres
|
||||
commands:
|
||||
- id
|
||||
- psql --version
|
||||
- sleep 5
|
||||
- pg_isready -h database -U postgres -d dspacestatistics
|
||||
- createuser -h database -U postgres dspacestatistics
|
||||
- psql -h database -U postgres -c "ALTER USER dspacestatistics WITH PASSWORD 'dspacestatistics'"
|
||||
- psql -h database -U postgres -d dspacestatistics < tests/dspacestatistics.sql
|
||||
|
||||
- name: test
|
||||
image: python:3.10-slim
|
||||
environment:
|
||||
PGPASSWORD: dspacestatistics
|
||||
DATABASE_HOST: database
|
||||
commands:
|
||||
- id
|
||||
- python -V
|
||||
- apt update && apt install -y gcc git libpq-dev
|
||||
- pip install -r requirements-dev.txt
|
||||
- pytest
|
||||
|
||||
services:
|
||||
- name: database
|
||||
image: postgres:10-alpine
|
||||
environment:
|
||||
POSTGRES_USER: postgres
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_DB: dspacestatistics
|
||||
|
||||
---
|
||||
kind: pipeline
|
||||
type: docker
|
||||
name: python39
|
||||
|
||||
steps:
|
||||
@ -24,7 +63,7 @@ steps:
|
||||
commands:
|
||||
- id
|
||||
- python -V
|
||||
- apt update && apt install -y gcc
|
||||
- apt update && apt install -y gcc git libpq-dev
|
||||
- pip install -r requirements-dev.txt
|
||||
- pytest
|
||||
|
||||
@ -71,6 +110,7 @@ steps:
|
||||
commands:
|
||||
- id
|
||||
- python -V
|
||||
- apt update && apt install -y gcc git libpq-dev
|
||||
- pip install -r requirements-dev.txt
|
||||
- pytest
|
||||
|
||||
@ -109,6 +149,7 @@ steps:
|
||||
commands:
|
||||
- id
|
||||
- python -V
|
||||
- apt update && apt install -y gcc git libpq-dev
|
||||
- pip install -r requirements-dev.txt
|
||||
- pytest
|
||||
|
||||
@ -147,6 +188,7 @@ steps:
|
||||
commands:
|
||||
- id
|
||||
- python -V
|
||||
- apt update && apt install -y gcc git libpq-dev
|
||||
- pip install -r requirements-dev.txt
|
||||
- pytest
|
||||
|
||||
|
61
.github/workflows/python-app.yml
vendored
Normal file
61
.github/workflows/python-app.yml
vendored
Normal file
@ -0,0 +1,61 @@
|
||||
# This workflow will install Python dependencies, run tests and lint with a single version of Python
|
||||
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
|
||||
|
||||
name: Build and Test
|
||||
|
||||
on: ['push', 'pull_request']
|
||||
|
||||
jobs:
|
||||
build:
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
services:
|
||||
database:
|
||||
image: postgres:10-alpine
|
||||
env:
|
||||
# password for postgres user in the Docker container
|
||||
POSTGRES_PASSWORD: postgres
|
||||
# default database to create
|
||||
POSTGRES_DB: dspacestatistics
|
||||
options: >-
|
||||
--health-cmd pg_isready
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
ports:
|
||||
- 5432:5432
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- name: Set up Python 3.10
|
||||
uses: actions/setup-python@v2
|
||||
with:
|
||||
python-version: '3.10'
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install flake8 pytest
|
||||
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
|
||||
if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
|
||||
- name: Lint with flake8
|
||||
run: |
|
||||
# stop the build if there are Python syntax errors or undefined names
|
||||
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
|
||||
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
|
||||
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
|
||||
- name: Set up PostgreSQL
|
||||
run: |
|
||||
pg_isready -U postgres -d dspacestatistics
|
||||
createuser -U postgres dspacestatistics
|
||||
psql -U postgres -c "ALTER USER dspacestatistics WITH PASSWORD 'dspacestatistics'"
|
||||
psql -U postgres -d dspacestatistics < tests/dspacestatistics.sql
|
||||
env:
|
||||
PGHOST: localhost
|
||||
PGPASSWORD: postgres
|
||||
- name: Test with pytest
|
||||
run: |
|
||||
pytest
|
||||
env:
|
||||
PGHOST: localhost
|
||||
PGPASSWORD: dspacestatistics
|
1
.gitignore
vendored
1
.gitignore
vendored
@ -1,2 +1,3 @@
|
||||
__pycache__
|
||||
venv
|
||||
*.egg-info
|
||||
|
32
CHANGELOG.md
32
CHANGELOG.md
@ -4,7 +4,32 @@ All notable changes to this project will be documented in this file.
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## Unreleased
|
||||
## 1.4.3 - 2022-03-26
|
||||
### Updated
|
||||
- Update dependencies with `poetry update`
|
||||
- Falcon 3.1.0, a minor change for us, but good to be using a current upstream
|
||||
version
|
||||
|
||||
## 1.4.2 - 2021-04-14
|
||||
### Updated
|
||||
- Update dependencies with `poetry update`
|
||||
- Falcon 3.0.0, a minor change for us, but good to be using a current upstream
|
||||
version
|
||||
|
||||
### Fixed
|
||||
- Bug in several of the "valid page" tests
|
||||
|
||||
### Added
|
||||
- GitHub Actions workflow to build and test the API
|
||||
|
||||
## [1.4.1] - 2021-01-14
|
||||
### Changed
|
||||
- Limit Solr query to UUIDs to avoid errors with unmigrated legacy stats (https://github.com/ilri/dspace-statistics-api/issues/12)
|
||||
|
||||
### Updated
|
||||
- Dev dependencies
|
||||
|
||||
## [1.4.0] - 2020-12-27
|
||||
### Added
|
||||
- indexer.py now indexes views and downloads for communities and collections
|
||||
- API endpoints for /communities, /community/id, /collections, and /collections/id
|
||||
@ -16,6 +41,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
deterministically
|
||||
- Use `fl` parameter in indexer to return only the field we are faceting by
|
||||
- Minor refactoring of imports for PEP8 style
|
||||
- More correct calculation of `totalPages` parameter in REST API response
|
||||
|
||||
## [1.3.2] - 2020-11-18
|
||||
### Fixed
|
||||
@ -71,7 +97,7 @@ and gunicorn 20.0.4
|
||||
- Minor syntax issues highlighted by flake8
|
||||
|
||||
## [1.1.0] - 2019-05-05
|
||||
## Updated
|
||||
### Updated
|
||||
- Falcon 2.0.0 (@alanorth)
|
||||
|
||||
## [1.0.0] - 2019-04-15
|
||||
@ -89,7 +115,7 @@ and gunicorn 20.0.4
|
||||
## [0.9.0] - 2019-01-22
|
||||
### Updated
|
||||
- pytest version 4.0.0
|
||||
- Fix indexing of sharded statistics cores ([#10))
|
||||
- Fix indexing of sharded statistics cores (#10)
|
||||
- Handle case of missing views/downloads gracefully
|
||||
|
||||
## [0.8.1] - 2018-11-14
|
||||
|
11
README.md
11
README.md
@ -1,8 +1,16 @@
|
||||
# DSpace Statistics API [](https://ci.mjanja.ch/alanorth/dspace-statistics-api) [](https://builds.sr.ht/~alanorth/dspace-statistics-api?)
|
||||
<h1 align="center">DSpace Statistics API</h1>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://ci.mjanja.ch/alanorth/dspace-statistics-api"><img alt="Build Status" src="https://ci.mjanja.ch/api/badges/alanorth/dspace-statistics-api/status.svg?ref=refs/heads/v6_x"></a>
|
||||
<a href="https://github.com/ilri/dspace-statistics-api/actions"><img alt="Build and Test" src="https://github.com/ilri/dspace-statistics-api/actions/workflows/python-app.yml/badge.svg"></a>
|
||||
<a href="https://github.com/psf/black"><img alt="Code style: black" src="https://img.shields.io/badge/code%20style-black-000000.svg"></a>
|
||||
</p>
|
||||
|
||||
DSpace stores item view and download events in a Solr "statistics" core. This information is available for use in the various DSpace user interfaces, but is not exposed externally via any APIs. The DSpace 4/5/6 [REST API](https://wiki.lyrasis.org/display/DSDOC5x/REST+API), for example, only exposes _metadata_ about communities, collections, items, and bitstreams.
|
||||
|
||||
- If your DSpace is version 4 or 5, use [dspace-statistics-api v1.1.1](https://github.com/ilri/dspace-statistics-api/releases/tag/v1.1.1)
|
||||
- If your DSpace is version 6+, use [dspace-statistics-api v1.2.0 or greater](https://github.com/ilri/dspace-statistics-api/releases/tag/v1.2.0)
|
||||
- Please make sure your statistics have been migrated from integers to UUIDs with the [solr-upgrade-statistics-6x](https://wiki.lyrasis.org/display/DSDOC6x/SOLR+Statistics+Maintenance) command
|
||||
|
||||
This project contains an indexer and a [Falcon-based](https://falcon.readthedocs.io/) web application to make the item, community, and collection statistics available via a simple REST API. You can read more about the Solr queries used to gather the item view and download statistics on the [DSpace wiki](https://wiki.lyrasis.org/display/DSPACE/Solr).
|
||||
|
||||
@ -119,7 +127,6 @@ The id is the *internal* UUID for an item, community, or collection. You can get
|
||||
- Use JSON in PostgreSQL
|
||||
- Add top items endpoint, perhaps `/top/items` or `/items/top`?
|
||||
- Actually we could add `/items?limit=10&sort=views`
|
||||
- Add Swagger with OpenAPI 3.0.x with [falcon-swagger-ui](https://github.com/rdidyk/falcon-swagger-ui)
|
||||
|
||||
## License
|
||||
This work is licensed under the [GPLv3](https://www.gnu.org/licenses/gpl-3.0.en.html).
|
||||
|
@ -1,4 +1,7 @@
|
||||
# SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
import json
|
||||
import math
|
||||
|
||||
import falcon
|
||||
import psycopg2.extras
|
||||
@ -16,9 +19,9 @@ class RootResource:
|
||||
resp.content_type = "text/html"
|
||||
docs_html = (
|
||||
"<!DOCTYPE html>"
|
||||
"<html lang=\"en-US\">"
|
||||
'<html lang="en-US">'
|
||||
" <head>"
|
||||
" <meta charset=\"UTF-8\">"
|
||||
' <meta charset="UTF-8">'
|
||||
" <title>DSpace Statistics API</title>"
|
||||
" </head>"
|
||||
" <body>"
|
||||
@ -28,7 +31,7 @@ class RootResource:
|
||||
"</html"
|
||||
)
|
||||
|
||||
resp.body = docs_html
|
||||
resp.text = docs_html
|
||||
|
||||
|
||||
class StatusResource:
|
||||
@ -57,7 +60,10 @@ class OpenAPIJSONResource:
|
||||
if DSPACE_STATISTICS_API_URL != "":
|
||||
data["servers"] = [{"url": DSPACE_STATISTICS_API_URL}]
|
||||
|
||||
resp.body = json.dumps(data)
|
||||
# Set the version in the schema so Swagger UI can display it
|
||||
data["info"]["version"] = VERSION
|
||||
|
||||
resp.text = json.dumps(data)
|
||||
|
||||
|
||||
class AllStatisticsResource:
|
||||
@ -75,7 +81,7 @@ class AllStatisticsResource:
|
||||
with db.cursor() as cursor:
|
||||
# get total number of communities/collections/items so we can estimate the pages
|
||||
cursor.execute(f"SELECT COUNT(id) FROM {req.context.statistics_scope}")
|
||||
pages = round(cursor.fetchone()[0] / limit)
|
||||
pages = math.ceil(cursor.fetchone()[0] / limit)
|
||||
|
||||
# get statistics and use limit and offset to page through results
|
||||
cursor.execute(
|
||||
@ -128,7 +134,7 @@ class AllStatisticsResource:
|
||||
# Helper variables to make working with pages/items/results easier and
|
||||
# to make the code easier to understand
|
||||
number_of_elements: int = len(req.context.elements)
|
||||
pages: int = int(number_of_elements / req.context.limit)
|
||||
pages: int = math.ceil(number_of_elements / req.context.limit)
|
||||
first_element: int = req.context.page * req.context.limit
|
||||
last_element: int = first_element + req.context.limit
|
||||
# Get a subset of the POSTed items based on our limit. Note that Python
|
||||
@ -209,24 +215,24 @@ class SingleStatisticsResource:
|
||||
resp.media = statistics
|
||||
|
||||
|
||||
api = application = falcon.API()
|
||||
api.add_route("/", RootResource())
|
||||
api.add_route("/status", StatusResource())
|
||||
app = application = falcon.App()
|
||||
app.add_route("/", RootResource())
|
||||
app.add_route("/status", StatusResource())
|
||||
|
||||
# Item routes
|
||||
api.add_route("/items", AllStatisticsResource())
|
||||
api.add_route("/item/{id_:uuid}", SingleStatisticsResource())
|
||||
app.add_route("/items", AllStatisticsResource())
|
||||
app.add_route("/item/{id_:uuid}", SingleStatisticsResource())
|
||||
|
||||
# Community routes
|
||||
api.add_route("/communities", AllStatisticsResource())
|
||||
api.add_route("/community/{id_:uuid}", SingleStatisticsResource())
|
||||
app.add_route("/communities", AllStatisticsResource())
|
||||
app.add_route("/community/{id_:uuid}", SingleStatisticsResource())
|
||||
|
||||
# Collection routes
|
||||
api.add_route("/collections", AllStatisticsResource())
|
||||
api.add_route("/collection/{id_:uuid}", SingleStatisticsResource())
|
||||
app.add_route("/collections", AllStatisticsResource())
|
||||
app.add_route("/collection/{id_:uuid}", SingleStatisticsResource())
|
||||
|
||||
# Route to the Swagger UI OpenAPI schema
|
||||
api.add_route("/docs/openapi.json", OpenAPIJSONResource())
|
||||
# Route to the Swagger UI Openapp schema
|
||||
app.add_route("/docs/openapi.json", OpenAPIJSONResource())
|
||||
|
||||
# Path to host the Swagger UI. Keep in mind that Falcon will add a route for
|
||||
# this automatically when we register Swagger and the path will be relative
|
||||
@ -236,12 +242,12 @@ SWAGGERUI_PATH = "/swagger"
|
||||
# The *absolute* path to the OpenJSON schema. This must be absolute because
|
||||
# it will be requested by the client and must resolve absolutely. Note: the
|
||||
# name of this variable is misleading because it is actually the schema URL
|
||||
# but we pass it into the register_swaggerui_app() function as the api_url
|
||||
# but we pass it into the register_swaggerui_app() function as the app_url
|
||||
# parameter.
|
||||
SWAGGERUI_API_URL = f"{DSPACE_STATISTICS_API_URL}/docs/openapi.json"
|
||||
|
||||
register_swaggerui_app(
|
||||
api,
|
||||
app,
|
||||
SWAGGERUI_PATH,
|
||||
SWAGGERUI_API_URL,
|
||||
config={
|
||||
|
@ -1,3 +1,5 @@
|
||||
# SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
import os
|
||||
|
||||
# Check if Solr connection information was provided in the environment
|
||||
@ -16,6 +18,6 @@ DATABASE_PORT = os.environ.get("DATABASE_PORT", "5432")
|
||||
# the vanilla DSpace REST API.
|
||||
DSPACE_STATISTICS_API_URL = os.environ.get("DSPACE_STATISTICS_API_URL", "")
|
||||
|
||||
VERSION = "1.4.0-dev"
|
||||
VERSION = "1.4.3"
|
||||
|
||||
# vim: set sw=4 ts=4 expandtab:
|
||||
|
@ -1,3 +1,5 @@
|
||||
# SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
import falcon
|
||||
import psycopg2
|
||||
import psycopg2.extras
|
||||
|
@ -1,7 +1,7 @@
|
||||
{
|
||||
"openapi": "3.0.3",
|
||||
"info": {
|
||||
"version": "1.4.0-dev",
|
||||
"version": "1.4.3",
|
||||
"title": "DSpace Statistics API",
|
||||
"description": "A [Falcon-based](https://falcon.readthedocs.io/) web application to make DSpace's item, community, and collection statistics available via a simple REST API. This Swagger interface is powered by [falcon-swagger-ui](https://github.com/rdidyk/falcon-swagger-ui).",
|
||||
"license": {
|
||||
@ -142,7 +142,7 @@
|
||||
},
|
||||
"example": {
|
||||
"limit": 100,
|
||||
"page": 5,
|
||||
"page": 0,
|
||||
"dateFrom": "2020-01-01T00:00:00Z",
|
||||
"dateTo": "2020-12-31T00:00:00Z",
|
||||
"items": [
|
||||
@ -502,7 +502,7 @@
|
||||
},
|
||||
"example": {
|
||||
"limit": 100,
|
||||
"page": 2,
|
||||
"page": 0,
|
||||
"dateFrom": "2020-01-01T00:00:00Z",
|
||||
"dateTo": "2020-12-31T00:00:00Z",
|
||||
"collections": [
|
||||
@ -613,4 +613,4 @@
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -1,23 +1,7 @@
|
||||
# SPDX-License-Identifier: GPL-3.0-only
|
||||
#
|
||||
# indexer.py
|
||||
#
|
||||
# Copyright 2018 Alan Orth.
|
||||
#
|
||||
# This program is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
# ---
|
||||
#
|
||||
# Connects to a DSpace Solr statistics core and ingests views and downloads for
|
||||
# communities, collections, and items into a PostgreSQL database.
|
||||
#
|
||||
@ -28,6 +12,8 @@
|
||||
#
|
||||
# See: https://wiki.duraspace.org/display/DSPACE/Solr
|
||||
|
||||
import math
|
||||
|
||||
import psycopg2.extras
|
||||
import requests
|
||||
|
||||
@ -45,7 +31,7 @@ def index_views(indexType: str, facetField: str):
|
||||
#
|
||||
# see: https://lucene.apache.org/solr/guide/6_6/the-stats-component.html
|
||||
solr_query_params = {
|
||||
"q": "type:2",
|
||||
"q": f"type:2 AND {facetField}:/.{{36}}/",
|
||||
"fq": "-isBot:true AND statistics_type:view",
|
||||
"fl": facetField,
|
||||
"facet": "true",
|
||||
@ -75,9 +61,9 @@ def index_views(indexType: str, facetField: str):
|
||||
|
||||
exit(0)
|
||||
|
||||
# divide results into "pages" (cast to int to effectively round down)
|
||||
# divide results into "pages" and round up to next integer
|
||||
results_per_page = 100
|
||||
results_num_pages = int(results_totalNumFacets / results_per_page)
|
||||
results_num_pages = math.ceil(results_totalNumFacets / results_per_page)
|
||||
results_current_page = 0
|
||||
|
||||
with DatabaseManager() as db:
|
||||
@ -92,7 +78,7 @@ def index_views(indexType: str, facetField: str):
|
||||
)
|
||||
|
||||
solr_query_params = {
|
||||
"q": "type:2",
|
||||
"q": f"type:2 AND {facetField}:/.{{36}}/",
|
||||
"fq": "-isBot:true AND statistics_type:view",
|
||||
"fl": facetField,
|
||||
"facet": "true",
|
||||
@ -128,7 +114,7 @@ def index_views(indexType: str, facetField: str):
|
||||
def index_downloads(indexType: str, facetField: str):
|
||||
# get the total number of distinct facets for items with at least 1 download
|
||||
solr_query_params = {
|
||||
"q": "type:0",
|
||||
"q": f"type:0 AND {facetField}:/.{{36}}/",
|
||||
"fq": "-isBot:true AND statistics_type:view AND bundleName:ORIGINAL",
|
||||
"fl": facetField,
|
||||
"facet": "true",
|
||||
@ -158,9 +144,8 @@ def index_downloads(indexType: str, facetField: str):
|
||||
|
||||
exit(0)
|
||||
|
||||
# divide results into "pages" (cast to int to effectively round down)
|
||||
results_per_page = 100
|
||||
results_num_pages = int(results_totalNumFacets / results_per_page)
|
||||
results_num_pages = math.ceil(results_totalNumFacets / results_per_page)
|
||||
results_current_page = 0
|
||||
|
||||
with DatabaseManager() as db:
|
||||
@ -175,7 +160,7 @@ def index_downloads(indexType: str, facetField: str):
|
||||
)
|
||||
|
||||
solr_query_params = {
|
||||
"q": "type:0",
|
||||
"q": f"type:0 AND {facetField}:/.{{36}}/",
|
||||
"fq": "-isBot:true AND statistics_type:view AND bundleName:ORIGINAL",
|
||||
"fl": facetField,
|
||||
"facet": "true",
|
||||
|
@ -1,3 +1,5 @@
|
||||
# SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
import requests
|
||||
|
||||
from .config import SOLR_SERVER
|
||||
|
@ -1,3 +1,5 @@
|
||||
# SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
import datetime
|
||||
import json
|
||||
import re
|
||||
|
964
poetry.lock
generated
964
poetry.lock
generated
File diff suppressed because it is too large
Load Diff
@ -1,6 +1,6 @@
|
||||
[tool.poetry]
|
||||
name = "dspace-statistics-api"
|
||||
version = "1.4.0-dev"
|
||||
version = "1.4.3"
|
||||
description = "A simple REST API to expose Solr view and download statistics for items, communities, and collections in a DSpace repository."
|
||||
authors = ["Alan Orth <aorth@mjanja.ch>"]
|
||||
license = "GPL-3.0-only"
|
||||
@ -8,17 +8,16 @@ license = "GPL-3.0-only"
|
||||
[tool.poetry.dependencies]
|
||||
python = "^3.6"
|
||||
gunicorn = "^20.0.4"
|
||||
falcon = "^2.0.0"
|
||||
psycopg2-binary = "^2.8.6"
|
||||
falcon = "3.1.0"
|
||||
psycopg2 = "^2.9.1"
|
||||
requests = "^2.24.0"
|
||||
falcon-swagger-ui = {git = "https://github.com/alanorth/falcon-swagger-ui.git"}
|
||||
falcon-swagger-ui = {git = "https://github.com/alanorth/falcon-swagger-ui.git", rev="falcon3-update-swagger-ui"}
|
||||
|
||||
[tool.poetry.dev-dependencies]
|
||||
ipython = { version = "^7.18.1", python = "^3.7" }
|
||||
flake8 = "^3.8.4"
|
||||
flake8 = "^4.0.1"
|
||||
pytest = "^6.1.1"
|
||||
isort = "^5.5.4"
|
||||
black = "^20.8b1"
|
||||
black = {version = "^22.1.0", python = ">=3.6.2"}
|
||||
isort = {version = "^5.9.1", python = ">=3.6.1"}
|
||||
|
||||
[build-system]
|
||||
requires = ["poetry>=0.12"]
|
||||
|
@ -1,51 +1,37 @@
|
||||
appdirs==1.4.4
|
||||
appnope==0.1.2; python_version >= "3.7" and python_version < "4.0" and sys_platform == "darwin"
|
||||
atomicwrites==1.4.0; sys_platform == "win32"
|
||||
attrs==20.3.0
|
||||
backcall==0.2.0; python_version >= "3.7" and python_version < "4.0"
|
||||
black==20.8b1
|
||||
certifi==2020.12.5
|
||||
chardet==4.0.0
|
||||
click==7.1.2
|
||||
colorama==0.4.4; python_version >= "3.7" and python_version < "4.0" and sys_platform == "win32" or sys_platform == "win32"
|
||||
dataclasses==0.6; python_version < "3.7"
|
||||
decorator==4.4.2; python_version >= "3.7" and python_version < "4.0"
|
||||
falcon==2.0.0
|
||||
-e git+https://github.com/alanorth/falcon-swagger-ui.git@a44244c85dceccfcd249b62fea4ee82a8221e3d2#egg=falcon-swagger-ui
|
||||
flake8==3.8.4
|
||||
gunicorn==20.0.4
|
||||
idna==2.10
|
||||
importlib-metadata==3.3.0; python_version < "3.8"
|
||||
iniconfig==1.1.1
|
||||
ipython==7.19.0; python_version >= "3.7" and python_version < "4.0"
|
||||
ipython-genutils==0.2.0; python_version >= "3.7" and python_version < "4.0"
|
||||
isort==5.6.4
|
||||
jedi==0.17.2; python_version >= "3.7" and python_version < "4.0"
|
||||
jinja2==2.11.2
|
||||
markupsafe==1.1.1
|
||||
mccabe==0.6.1
|
||||
mypy-extensions==0.4.3
|
||||
packaging==20.8
|
||||
parso==0.7.1; python_version >= "3.7" and python_version < "4.0"
|
||||
pathspec==0.8.1
|
||||
pexpect==4.8.0; python_version >= "3.7" and python_version < "4.0" and sys_platform != "win32"
|
||||
pickleshare==0.7.5; python_version >= "3.7" and python_version < "4.0"
|
||||
pluggy==0.13.1
|
||||
prompt-toolkit==3.0.8; python_version >= "3.7" and python_version < "4.0"
|
||||
psycopg2-binary==2.8.6
|
||||
ptyprocess==0.6.0; python_version >= "3.7" and python_version < "4.0" and sys_platform != "win32"
|
||||
py==1.10.0
|
||||
pycodestyle==2.6.0
|
||||
pyflakes==2.2.0
|
||||
pygments==2.7.3; python_version >= "3.7" and python_version < "4.0"
|
||||
pyparsing==2.4.7
|
||||
pytest==6.2.1
|
||||
regex==2020.11.13
|
||||
requests==2.25.1
|
||||
toml==0.10.2
|
||||
traitlets==5.0.5; python_version >= "3.7" and python_version < "4.0"
|
||||
typed-ast==1.4.1
|
||||
typing-extensions==3.7.4.3
|
||||
urllib3==1.26.2
|
||||
wcwidth==0.2.5; python_version >= "3.7" and python_version < "4.0"
|
||||
zipp==3.4.0; python_version < "3.8"
|
||||
atomicwrites==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" and sys_platform == "win32" or sys_platform == "win32" and python_version >= "3.6" and python_full_version >= "3.4.0"
|
||||
attrs==21.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
|
||||
black==22.1.0; python_full_version >= "3.6.2"
|
||||
certifi==2021.10.8; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0"
|
||||
charset-normalizer==2.0.12; python_full_version >= "3.6.0" and python_version >= "3"
|
||||
click==8.0.4; python_version >= "3.6" and python_full_version >= "3.6.2"
|
||||
colorama==0.4.4; sys_platform == "win32" and python_version >= "3.6" and python_full_version >= "3.6.2" and platform_system == "Windows"
|
||||
dataclasses==0.8; python_version >= "3.6" and python_version < "3.7" and python_full_version >= "3.6.2"
|
||||
falcon-swagger-ui @ git+https://github.com/alanorth/falcon-swagger-ui.git@falcon3-update-swagger-ui
|
||||
falcon==3.1.0; python_version >= "3.5"
|
||||
flake8==4.0.1; python_version >= "3.6"
|
||||
gunicorn==20.1.0; python_version >= "3.5"
|
||||
idna==3.3; python_version >= "3.5" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.5"
|
||||
importlib-metadata==4.2.0; python_version < "3.8" and python_version >= "3.6" and python_full_version >= "3.6.2"
|
||||
iniconfig==1.1.1; python_version >= "3.6"
|
||||
isort==5.10.1; python_full_version >= "3.6.1"
|
||||
jinja2==3.0.3; python_version >= "3.6"
|
||||
markupsafe==2.0.1; python_version >= "3.6"
|
||||
mccabe==0.6.1; python_version >= "3.6"
|
||||
mypy-extensions==0.4.3; python_full_version >= "3.6.2"
|
||||
packaging==21.3; python_version >= "3.6"
|
||||
pathspec==0.9.0; python_full_version >= "3.6.2"
|
||||
platformdirs==2.4.0; python_version >= "3.6" and python_full_version >= "3.6.2"
|
||||
pluggy==1.0.0; python_version >= "3.6"
|
||||
psycopg2==2.9.3; python_version >= "3.6"
|
||||
py==1.11.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
|
||||
pycodestyle==2.8.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6"
|
||||
pyflakes==2.4.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6"
|
||||
pyparsing==3.0.7; python_version >= "3.6"
|
||||
pytest==6.2.5; python_version >= "3.6"
|
||||
requests==2.27.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0")
|
||||
toml==0.10.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6"
|
||||
tomli==1.2.3; python_version >= "3.6" and python_full_version >= "3.6.2"
|
||||
typed-ast==1.5.2; python_version < "3.8" and implementation_name == "cpython" and python_full_version >= "3.6.2" and python_version >= "3.6"
|
||||
typing-extensions==4.1.1; python_version < "3.8" and python_full_version >= "3.6.2" and python_version >= "3.6"
|
||||
urllib3==1.26.9; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "4"
|
||||
zipp==3.6.0; python_version < "3.8" and python_version >= "3.6"
|
||||
|
@ -1,11 +1,11 @@
|
||||
certifi==2020.12.5
|
||||
chardet==4.0.0
|
||||
falcon==2.0.0
|
||||
-e git+https://github.com/alanorth/falcon-swagger-ui.git@a44244c85dceccfcd249b62fea4ee82a8221e3d2#egg=falcon-swagger-ui
|
||||
gunicorn==20.0.4
|
||||
idna==2.10
|
||||
jinja2==2.11.2
|
||||
markupsafe==1.1.1
|
||||
psycopg2-binary==2.8.6
|
||||
requests==2.25.1
|
||||
urllib3==1.26.2
|
||||
certifi==2021.10.8; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0"
|
||||
charset-normalizer==2.0.12; python_full_version >= "3.6.0" and python_version >= "3"
|
||||
falcon-swagger-ui @ git+https://github.com/alanorth/falcon-swagger-ui.git@falcon3-update-swagger-ui
|
||||
falcon==3.1.0; python_version >= "3.5"
|
||||
gunicorn==20.1.0; python_version >= "3.5"
|
||||
idna==3.3; python_version >= "3.5" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.5"
|
||||
jinja2==3.0.3; python_version >= "3.6"
|
||||
markupsafe==2.0.1; python_version >= "3.6"
|
||||
psycopg2==2.9.3; python_version >= "3.6"
|
||||
requests==2.27.1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0")
|
||||
urllib3==1.26.9; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "4"
|
||||
|
@ -1,14 +1,17 @@
|
||||
from falcon import testing
|
||||
# SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
import json
|
||||
import pytest
|
||||
from unittest.mock import patch
|
||||
|
||||
from dspace_statistics_api.app import api
|
||||
import pytest
|
||||
from falcon import testing
|
||||
|
||||
from dspace_statistics_api.app import app
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
return testing.TestClient(api)
|
||||
return testing.TestClient(app)
|
||||
|
||||
|
||||
def test_get_collection(client):
|
||||
@ -309,7 +312,7 @@ def test_post_collections_valid_page(client):
|
||||
assert response.status_code == 200
|
||||
assert response.json["limit"] == 100
|
||||
assert response.json["currentPage"] == 0
|
||||
assert response.json["totalPages"] == 0
|
||||
assert response.json["totalPages"] == 1
|
||||
assert len(response.json["statistics"]) == 2
|
||||
assert isinstance(response.json["statistics"][0]["views"], int)
|
||||
assert isinstance(response.json["statistics"][0]["downloads"], int)
|
||||
|
@ -1,14 +1,17 @@
|
||||
from falcon import testing
|
||||
# SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
import json
|
||||
import pytest
|
||||
from unittest.mock import patch
|
||||
|
||||
from dspace_statistics_api.app import api
|
||||
import pytest
|
||||
from falcon import testing
|
||||
|
||||
from dspace_statistics_api.app import app
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
return testing.TestClient(api)
|
||||
return testing.TestClient(app)
|
||||
|
||||
|
||||
def test_get_community(client):
|
||||
@ -309,7 +312,7 @@ def test_post_communities_valid_page(client):
|
||||
assert response.status_code == 200
|
||||
assert response.json["limit"] == 100
|
||||
assert response.json["currentPage"] == 0
|
||||
assert response.json["totalPages"] == 0
|
||||
assert response.json["totalPages"] == 1
|
||||
assert len(response.json["statistics"]) == 2
|
||||
assert isinstance(response.json["statistics"][0]["views"], int)
|
||||
assert isinstance(response.json["statistics"][0]["downloads"], int)
|
||||
|
@ -1,12 +1,14 @@
|
||||
from falcon import testing
|
||||
import pytest
|
||||
# SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
from dspace_statistics_api.app import api
|
||||
import pytest
|
||||
from falcon import testing
|
||||
|
||||
from dspace_statistics_api.app import app
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
return testing.TestClient(api)
|
||||
return testing.TestClient(app)
|
||||
|
||||
|
||||
def test_get_docs(client):
|
||||
|
@ -1,14 +1,17 @@
|
||||
from falcon import testing
|
||||
# SPDX-License-Identifier: GPL-3.0-only
|
||||
|
||||
import json
|
||||
import pytest
|
||||
from unittest.mock import patch
|
||||
|
||||
from dspace_statistics_api.app import api
|
||||
import pytest
|
||||
from falcon import testing
|
||||
|
||||
from dspace_statistics_api.app import app
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
return testing.TestClient(api)
|
||||
return testing.TestClient(app)
|
||||
|
||||
|
||||
def test_get_item(client):
|
||||
@ -309,7 +312,7 @@ def test_post_items_valid_page(client):
|
||||
assert response.status_code == 200
|
||||
assert response.json["limit"] == 100
|
||||
assert response.json["currentPage"] == 0
|
||||
assert response.json["totalPages"] == 0
|
||||
assert response.json["totalPages"] == 1
|
||||
assert len(response.json["statistics"]) == 2
|
||||
assert isinstance(response.json["statistics"][0]["views"], int)
|
||||
assert isinstance(response.json["statistics"][0]["downloads"], int)
|
||||
|
Reference in New Issue
Block a user