mirror of
				https://github.com/ilri/dspace-statistics-api.git
				synced 2025-10-31 12:51:15 +01:00 
			
		
		
		
	Compare commits
	
		
			47 Commits
		
	
	
		
			v1.4.0
			...
			4c5326a176
		
	
	| Author | SHA1 | Date | |
|---|---|---|---|
| 4c5326a176 | |||
| 3b1ccafab4 | |||
| 58b5ae82d3 | |||
| 562aaeef7d | |||
| 5cdba6acb1 | |||
| dd0937179c | |||
| f0c6c004db | |||
| 6843f0a8ac | |||
| f5fcfcc05a | |||
| e8ac74b6d1 | |||
| 14fc14daee | |||
| 871aae537a | |||
| 2fada6c6ff | |||
| ef0991e352 | |||
| 4502d6053c | |||
| a524068cf6 | |||
| 964d5dff06 | |||
| a9252d1771 | |||
| a63687d516 | |||
| 73dc3a292e | |||
| 1e742bad41 | |||
| 164008981e | |||
| dd1769b954 | |||
| b009820fb4 | |||
| 9830295978 | |||
| c93a4d7455 | |||
| 2f8e4f8a0a | |||
| 0650c5985e | |||
| d814f1c4f0 | |||
| 00f30591c4 | |||
| acfe87b91a | |||
| bc6d84dda2 | |||
| 889fb2f74a | |||
| c42cd7a818 | |||
| f8bba59d66 | |||
| b8cb752a29 | |||
| 09496aa2b5 | |||
| ff5dc7506d | |||
| 80a11ead97 | |||
| a282c95933 | |||
| fd7cc36306 | |||
| a20ff09570 | |||
| fdc0e73088 | |||
| b15afc9f39 | |||
| 2bc18ef719 | |||
| 49751b53f0 | |||
| d1c177e146 | 
							
								
								
									
										21
									
								
								.build.yml
									
									
									
									
									
								
							
							
						
						
									
										21
									
								
								.build.yml
									
									
									
									
									
								
							| @@ -1,21 +0,0 @@ | |||||||
| image: archlinux |  | ||||||
| packages: |  | ||||||
|   - python-poetry |  | ||||||
|   - postgresql |  | ||||||
| sources: |  | ||||||
|   - https://git.sr.ht/~alanorth/dspace-statistics-api |  | ||||||
| tasks: |  | ||||||
|   - setup: | |  | ||||||
|       id |  | ||||||
|       psql --version |  | ||||||
|       sudo su - postgres -c "initdb --locale en_US.UTF-8 -E UTF8 -D '/var/lib/postgres/data'" |  | ||||||
|       sudo systemctl start postgresql |  | ||||||
|       createuser -U postgres dspacestatistics |  | ||||||
|       psql -U postgres -c "ALTER USER dspacestatistics WITH PASSWORD 'dspacestatistics'" |  | ||||||
|       createdb -U postgres -O dspacestatistics --encoding=UNICODE dspacestatistics |  | ||||||
|       cd dspace-statistics-api |  | ||||||
|       psql -U postgres -d dspacestatistics < tests/dspacestatistics.sql |  | ||||||
|       poetry install --no-root |  | ||||||
|   - test: | |  | ||||||
|       cd dspace-statistics-api |  | ||||||
|       poetry run pytest |  | ||||||
							
								
								
									
										44
									
								
								.drone.yml
									
									
									
									
									
								
							
							
						
						
									
										44
									
								
								.drone.yml
									
									
									
									
									
								
							| @@ -1,5 +1,44 @@ | |||||||
| kind: pipeline | kind: pipeline | ||||||
| type: docker | type: docker | ||||||
|  | name: python10 | ||||||
|  |  | ||||||
|  | steps: | ||||||
|  | - name: setup | ||||||
|  |   image: postgres:10-alpine | ||||||
|  |   environment: | ||||||
|  |     PGPASSWORD: postgres | ||||||
|  |   commands: | ||||||
|  |   - id | ||||||
|  |   - psql --version | ||||||
|  |   - sleep 5 | ||||||
|  |   - pg_isready -h database -U postgres -d dspacestatistics | ||||||
|  |   - createuser -h database -U postgres dspacestatistics | ||||||
|  |   - psql -h database -U postgres -c "ALTER USER dspacestatistics WITH PASSWORD 'dspacestatistics'" | ||||||
|  |   - psql -h database -U postgres -d dspacestatistics < tests/dspacestatistics.sql | ||||||
|  |  | ||||||
|  | - name: test | ||||||
|  |   image: python:3.10-slim | ||||||
|  |   environment: | ||||||
|  |     PGPASSWORD: dspacestatistics | ||||||
|  |     DATABASE_HOST: database | ||||||
|  |   commands: | ||||||
|  |   - id | ||||||
|  |   - python -V | ||||||
|  |   - apt update && apt install -y gcc git libpq-dev | ||||||
|  |   - pip install -r requirements-dev.txt | ||||||
|  |   - pytest | ||||||
|  |  | ||||||
|  | services: | ||||||
|  | - name: database | ||||||
|  |   image: postgres:10-alpine | ||||||
|  |   environment: | ||||||
|  |     POSTGRES_USER: postgres | ||||||
|  |     POSTGRES_PASSWORD: postgres | ||||||
|  |     POSTGRES_DB: dspacestatistics | ||||||
|  |  | ||||||
|  | --- | ||||||
|  | kind: pipeline | ||||||
|  | type: docker | ||||||
| name: python39 | name: python39 | ||||||
|  |  | ||||||
| steps: | steps: | ||||||
| @@ -24,7 +63,7 @@ steps: | |||||||
|   commands: |   commands: | ||||||
|   - id |   - id | ||||||
|   - python -V |   - python -V | ||||||
|   - apt update && apt install -y gcc |   - apt update && apt install -y gcc git libpq-dev | ||||||
|   - pip install -r requirements-dev.txt |   - pip install -r requirements-dev.txt | ||||||
|   - pytest |   - pytest | ||||||
|  |  | ||||||
| @@ -71,6 +110,7 @@ steps: | |||||||
|   commands: |   commands: | ||||||
|   - id |   - id | ||||||
|   - python -V |   - python -V | ||||||
|  |   - apt update && apt install -y gcc git libpq-dev | ||||||
|   - pip install -r requirements-dev.txt |   - pip install -r requirements-dev.txt | ||||||
|   - pytest |   - pytest | ||||||
|  |  | ||||||
| @@ -109,6 +149,7 @@ steps: | |||||||
|   commands: |   commands: | ||||||
|   - id |   - id | ||||||
|   - python -V |   - python -V | ||||||
|  |   - apt update && apt install -y gcc git libpq-dev | ||||||
|   - pip install -r requirements-dev.txt |   - pip install -r requirements-dev.txt | ||||||
|   - pytest |   - pytest | ||||||
|  |  | ||||||
| @@ -147,6 +188,7 @@ steps: | |||||||
|   commands: |   commands: | ||||||
|   - id |   - id | ||||||
|   - python -V |   - python -V | ||||||
|  |   - apt update && apt install -y gcc git libpq-dev | ||||||
|   - pip install -r requirements-dev.txt |   - pip install -r requirements-dev.txt | ||||||
|   - pytest |   - pytest | ||||||
|  |  | ||||||
|   | |||||||
							
								
								
									
										61
									
								
								.github/workflows/python-app.yml
									
									
									
									
										vendored
									
									
										Normal file
									
								
							
							
						
						
									
										61
									
								
								.github/workflows/python-app.yml
									
									
									
									
										vendored
									
									
										Normal file
									
								
							| @@ -0,0 +1,61 @@ | |||||||
|  | # This workflow will install Python dependencies, run tests and lint with a single version of Python | ||||||
|  | # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions | ||||||
|  |  | ||||||
|  | name: Build and Test | ||||||
|  |  | ||||||
|  | on: ['push', 'pull_request'] | ||||||
|  |  | ||||||
|  | jobs: | ||||||
|  |   build: | ||||||
|  |  | ||||||
|  |     runs-on: ubuntu-latest | ||||||
|  |  | ||||||
|  |     services: | ||||||
|  |       database: | ||||||
|  |         image: postgres:10-alpine | ||||||
|  |         env: | ||||||
|  |           # password for postgres user in the Docker container | ||||||
|  |           POSTGRES_PASSWORD: postgres | ||||||
|  |           # default database to create | ||||||
|  |           POSTGRES_DB: dspacestatistics | ||||||
|  |         options: >- | ||||||
|  |           --health-cmd pg_isready | ||||||
|  |           --health-interval 10s | ||||||
|  |           --health-timeout 5s | ||||||
|  |           --health-retries 5 | ||||||
|  |         ports: | ||||||
|  |           - 5432:5432 | ||||||
|  |  | ||||||
|  |     steps: | ||||||
|  |     - uses: actions/checkout@v2 | ||||||
|  |     - name: Set up Python 3.9 | ||||||
|  |       uses: actions/setup-python@v2 | ||||||
|  |       with: | ||||||
|  |         python-version: 3.9 | ||||||
|  |     - name: Install dependencies | ||||||
|  |       run: | | ||||||
|  |         python -m pip install --upgrade pip | ||||||
|  |         pip install flake8 pytest | ||||||
|  |         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi | ||||||
|  |         if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi | ||||||
|  |     - name: Lint with flake8 | ||||||
|  |       run: | | ||||||
|  |         # stop the build if there are Python syntax errors or undefined names | ||||||
|  |         flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics | ||||||
|  |         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide | ||||||
|  |         flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics | ||||||
|  |     - name: Set up PostgreSQL | ||||||
|  |       run: | | ||||||
|  |         pg_isready -U postgres -d dspacestatistics | ||||||
|  |         createuser -U postgres dspacestatistics | ||||||
|  |         psql -U postgres -c "ALTER USER dspacestatistics WITH PASSWORD 'dspacestatistics'" | ||||||
|  |         psql -U postgres -d dspacestatistics < tests/dspacestatistics.sql | ||||||
|  |       env: | ||||||
|  |         PGHOST: localhost | ||||||
|  |         PGPASSWORD: postgres | ||||||
|  |     - name: Test with pytest | ||||||
|  |       run: | | ||||||
|  |         pytest | ||||||
|  |       env: | ||||||
|  |         PGHOST: localhost | ||||||
|  |         PGPASSWORD: dspacestatistics | ||||||
							
								
								
									
										1
									
								
								.gitignore
									
									
									
									
										vendored
									
									
								
							
							
						
						
									
										1
									
								
								.gitignore
									
									
									
									
										vendored
									
									
								
							| @@ -1,2 +1,3 @@ | |||||||
| __pycache__ | __pycache__ | ||||||
| venv | venv | ||||||
|  | *.egg-info | ||||||
|   | |||||||
							
								
								
									
										23
									
								
								CHANGELOG.md
									
									
									
									
									
								
							
							
						
						
									
										23
									
								
								CHANGELOG.md
									
									
									
									
									
								
							| @@ -4,6 +4,25 @@ All notable changes to this project will be documented in this file. | |||||||
| The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), | ||||||
| and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). | ||||||
|  |  | ||||||
|  | ## 1.4.2 - 2021-04-14 | ||||||
|  | ### Updated | ||||||
|  | - Update dependencies with `poetry update` | ||||||
|  | - Falcon 3.0.0, a minor change for us, but good to be using a current upstream | ||||||
|  | version | ||||||
|  |  | ||||||
|  | ### Fixed | ||||||
|  | - Bug in several of the "valid page" tests | ||||||
|  |  | ||||||
|  | ### Added | ||||||
|  | - GitHub Actions workflow to build and test the API | ||||||
|  |  | ||||||
|  | ## [1.4.1] - 2021-01-14 | ||||||
|  | ### Changed | ||||||
|  | - Limit Solr query to UUIDs to avoid errors with unmigrated legacy stats (https://github.com/ilri/dspace-statistics-api/issues/12) | ||||||
|  |  | ||||||
|  | ### Updated | ||||||
|  | - Dev dependencies | ||||||
|  |  | ||||||
| ## [1.4.0] - 2020-12-27 | ## [1.4.0] - 2020-12-27 | ||||||
| ### Added | ### Added | ||||||
| - indexer.py now indexes views and downloads for communities and collections | - indexer.py now indexes views and downloads for communities and collections | ||||||
| @@ -72,7 +91,7 @@ and gunicorn 20.0.4 | |||||||
| - Minor syntax issues highlighted by flake8 | - Minor syntax issues highlighted by flake8 | ||||||
|  |  | ||||||
| ## [1.1.0] - 2019-05-05 | ## [1.1.0] - 2019-05-05 | ||||||
| ## Updated | ### Updated | ||||||
| - Falcon 2.0.0 (@alanorth) | - Falcon 2.0.0 (@alanorth) | ||||||
|  |  | ||||||
| ## [1.0.0] - 2019-04-15 | ## [1.0.0] - 2019-04-15 | ||||||
| @@ -90,7 +109,7 @@ and gunicorn 20.0.4 | |||||||
| ## [0.9.0] - 2019-01-22 | ## [0.9.0] - 2019-01-22 | ||||||
| ### Updated | ### Updated | ||||||
| - pytest version 4.0.0 | - pytest version 4.0.0 | ||||||
| - Fix indexing of sharded statistics cores ([#10)) | - Fix indexing of sharded statistics cores (#10) | ||||||
| - Handle case of missing views/downloads gracefully | - Handle case of missing views/downloads gracefully | ||||||
|  |  | ||||||
| ## [0.8.1] - 2018-11-14 | ## [0.8.1] - 2018-11-14 | ||||||
|   | |||||||
| @@ -1,8 +1,9 @@ | |||||||
| # DSpace Statistics API [](https://ci.mjanja.ch/alanorth/dspace-statistics-api) [](https://builds.sr.ht/~alanorth/dspace-statistics-api?) | # DSpace Statistics API [](https://ci.mjanja.ch/alanorth/dspace-statistics-api) [](https://github.com/ilri/dspace-statistics-api/actions/workflows/python-app.yml) | ||||||
| DSpace stores item view and download events in a Solr "statistics" core. This information is available for use in the various DSpace user interfaces, but is not exposed externally via any APIs. The DSpace 4/5/6 [REST API](https://wiki.lyrasis.org/display/DSDOC5x/REST+API), for example, only exposes _metadata_ about communities, collections, items, and bitstreams. | DSpace stores item view and download events in a Solr "statistics" core. This information is available for use in the various DSpace user interfaces, but is not exposed externally via any APIs. The DSpace 4/5/6 [REST API](https://wiki.lyrasis.org/display/DSDOC5x/REST+API), for example, only exposes _metadata_ about communities, collections, items, and bitstreams. | ||||||
|  |  | ||||||
| - If your DSpace is version 4 or 5, use [dspace-statistics-api v1.1.1](https://github.com/ilri/dspace-statistics-api/releases/tag/v1.1.1) | - If your DSpace is version 4 or 5, use [dspace-statistics-api v1.1.1](https://github.com/ilri/dspace-statistics-api/releases/tag/v1.1.1) | ||||||
| - If your DSpace is version 6+, use [dspace-statistics-api v1.2.0 or greater](https://github.com/ilri/dspace-statistics-api/releases/tag/v1.2.0) | - If your DSpace is version 6+, use [dspace-statistics-api v1.2.0 or greater](https://github.com/ilri/dspace-statistics-api/releases/tag/v1.2.0) | ||||||
|  |   - Please make sure your statistics have been migrated from integers to UUIDs with the [solr-upgrade-statistics-6x](https://wiki.lyrasis.org/display/DSDOC6x/SOLR+Statistics+Maintenance) command | ||||||
|  |  | ||||||
| This project contains an indexer and a [Falcon-based](https://falcon.readthedocs.io/) web application to make the item, community, and collection statistics available via a simple REST API. You can read more about the Solr queries used to gather the item view and download statistics on the [DSpace wiki](https://wiki.lyrasis.org/display/DSPACE/Solr). | This project contains an indexer and a [Falcon-based](https://falcon.readthedocs.io/) web application to make the item, community, and collection statistics available via a simple REST API. You can read more about the Solr queries used to gather the item view and download statistics on the [DSpace wiki](https://wiki.lyrasis.org/display/DSPACE/Solr). | ||||||
|  |  | ||||||
| @@ -119,7 +120,6 @@ The id is the *internal* UUID for an item, community, or collection. You can get | |||||||
| - Use JSON in PostgreSQL | - Use JSON in PostgreSQL | ||||||
| - Add top items endpoint, perhaps `/top/items` or `/items/top`? | - Add top items endpoint, perhaps `/top/items` or `/items/top`? | ||||||
|   - Actually we could add `/items?limit=10&sort=views` |   - Actually we could add `/items?limit=10&sort=views` | ||||||
| - Add Swagger with OpenAPI 3.0.x with [falcon-swagger-ui](https://github.com/rdidyk/falcon-swagger-ui) |  | ||||||
|  |  | ||||||
| ## License | ## License | ||||||
| This work is licensed under the [GPLv3](https://www.gnu.org/licenses/gpl-3.0.en.html). | This work is licensed under the [GPLv3](https://www.gnu.org/licenses/gpl-3.0.en.html). | ||||||
|   | |||||||
| @@ -1,3 +1,5 @@ | |||||||
|  | # SPDX-License-Identifier: GPL-3.0-only | ||||||
|  |  | ||||||
| import json | import json | ||||||
| import math | import math | ||||||
|  |  | ||||||
| @@ -29,7 +31,7 @@ class RootResource: | |||||||
|             "</html" |             "</html" | ||||||
|         ) |         ) | ||||||
|  |  | ||||||
|         resp.body = docs_html |         resp.text = docs_html | ||||||
|  |  | ||||||
|  |  | ||||||
| class StatusResource: | class StatusResource: | ||||||
| @@ -61,7 +63,7 @@ class OpenAPIJSONResource: | |||||||
|             # Set the version in the schema so Swagger UI can display it |             # Set the version in the schema so Swagger UI can display it | ||||||
|             data["info"]["version"] = VERSION |             data["info"]["version"] = VERSION | ||||||
|  |  | ||||||
|             resp.body = json.dumps(data) |             resp.text = json.dumps(data) | ||||||
|  |  | ||||||
|  |  | ||||||
| class AllStatisticsResource: | class AllStatisticsResource: | ||||||
| @@ -213,24 +215,24 @@ class SingleStatisticsResource: | |||||||
|                     resp.media = statistics |                     resp.media = statistics | ||||||
|  |  | ||||||
|  |  | ||||||
| api = application = falcon.API() | app = application = falcon.App() | ||||||
| api.add_route("/", RootResource()) | app.add_route("/", RootResource()) | ||||||
| api.add_route("/status", StatusResource()) | app.add_route("/status", StatusResource()) | ||||||
|  |  | ||||||
| # Item routes | # Item routes | ||||||
| api.add_route("/items", AllStatisticsResource()) | app.add_route("/items", AllStatisticsResource()) | ||||||
| api.add_route("/item/{id_:uuid}", SingleStatisticsResource()) | app.add_route("/item/{id_:uuid}", SingleStatisticsResource()) | ||||||
|  |  | ||||||
| # Community routes | # Community routes | ||||||
| api.add_route("/communities", AllStatisticsResource()) | app.add_route("/communities", AllStatisticsResource()) | ||||||
| api.add_route("/community/{id_:uuid}", SingleStatisticsResource()) | app.add_route("/community/{id_:uuid}", SingleStatisticsResource()) | ||||||
|  |  | ||||||
| # Collection routes | # Collection routes | ||||||
| api.add_route("/collections", AllStatisticsResource()) | app.add_route("/collections", AllStatisticsResource()) | ||||||
| api.add_route("/collection/{id_:uuid}", SingleStatisticsResource()) | app.add_route("/collection/{id_:uuid}", SingleStatisticsResource()) | ||||||
|  |  | ||||||
| # Route to the Swagger UI OpenAPI schema | # Route to the Swagger UI Openapp schema | ||||||
| api.add_route("/docs/openapi.json", OpenAPIJSONResource()) | app.add_route("/docs/openapi.json", OpenAPIJSONResource()) | ||||||
|  |  | ||||||
| # Path to host the Swagger UI. Keep in mind that Falcon will add a route for | # Path to host the Swagger UI. Keep in mind that Falcon will add a route for | ||||||
| # this automatically when we register Swagger and the path will be relative | # this automatically when we register Swagger and the path will be relative | ||||||
| @@ -240,12 +242,12 @@ SWAGGERUI_PATH = "/swagger" | |||||||
| # The *absolute* path to the OpenJSON schema. This must be absolute because | # The *absolute* path to the OpenJSON schema. This must be absolute because | ||||||
| # it will be requested by the client and must resolve absolutely. Note: the | # it will be requested by the client and must resolve absolutely. Note: the | ||||||
| # name of this variable is misleading because it is actually the schema URL | # name of this variable is misleading because it is actually the schema URL | ||||||
| # but we pass it into the register_swaggerui_app() function as the api_url | # but we pass it into the register_swaggerui_app() function as the app_url | ||||||
| # parameter. | # parameter. | ||||||
| SWAGGERUI_API_URL = f"{DSPACE_STATISTICS_API_URL}/docs/openapi.json" | SWAGGERUI_API_URL = f"{DSPACE_STATISTICS_API_URL}/docs/openapi.json" | ||||||
|  |  | ||||||
| register_swaggerui_app( | register_swaggerui_app( | ||||||
|     api, |     app, | ||||||
|     SWAGGERUI_PATH, |     SWAGGERUI_PATH, | ||||||
|     SWAGGERUI_API_URL, |     SWAGGERUI_API_URL, | ||||||
|     config={ |     config={ | ||||||
|   | |||||||
| @@ -1,3 +1,5 @@ | |||||||
|  | # SPDX-License-Identifier: GPL-3.0-only | ||||||
|  |  | ||||||
| import os | import os | ||||||
|  |  | ||||||
| # Check if Solr connection information was provided in the environment | # Check if Solr connection information was provided in the environment | ||||||
| @@ -16,6 +18,6 @@ DATABASE_PORT = os.environ.get("DATABASE_PORT", "5432") | |||||||
| # the vanilla DSpace REST API. | # the vanilla DSpace REST API. | ||||||
| DSPACE_STATISTICS_API_URL = os.environ.get("DSPACE_STATISTICS_API_URL", "") | DSPACE_STATISTICS_API_URL = os.environ.get("DSPACE_STATISTICS_API_URL", "") | ||||||
|  |  | ||||||
| VERSION = "1.4.0" | VERSION = "1.4.3-dev" | ||||||
|  |  | ||||||
| # vim: set sw=4 ts=4 expandtab: | # vim: set sw=4 ts=4 expandtab: | ||||||
|   | |||||||
| @@ -1,3 +1,5 @@ | |||||||
|  | # SPDX-License-Identifier: GPL-3.0-only | ||||||
|  |  | ||||||
| import falcon | import falcon | ||||||
| import psycopg2 | import psycopg2 | ||||||
| import psycopg2.extras | import psycopg2.extras | ||||||
|   | |||||||
| @@ -1,7 +1,7 @@ | |||||||
| { | { | ||||||
|   "openapi": "3.0.3", |   "openapi": "3.0.3", | ||||||
|   "info": { |   "info": { | ||||||
|     "version": "1.4.0", |     "version": "1.4.3-dev", | ||||||
|     "title": "DSpace Statistics API", |     "title": "DSpace Statistics API", | ||||||
|     "description": "A [Falcon-based](https://falcon.readthedocs.io/) web application to make DSpace's item, community, and collection statistics available via a simple REST API. This Swagger interface is powered by [falcon-swagger-ui](https://github.com/rdidyk/falcon-swagger-ui).", |     "description": "A [Falcon-based](https://falcon.readthedocs.io/) web application to make DSpace's item, community, and collection statistics available via a simple REST API. This Swagger interface is powered by [falcon-swagger-ui](https://github.com/rdidyk/falcon-swagger-ui).", | ||||||
|     "license": { |     "license": { | ||||||
|   | |||||||
| @@ -1,23 +1,7 @@ | |||||||
|  | # SPDX-License-Identifier: GPL-3.0-only | ||||||
| # | # | ||||||
| # indexer.py | # indexer.py | ||||||
| # | # | ||||||
| # Copyright 2018 Alan Orth. |  | ||||||
| # |  | ||||||
| # This program is free software: you can redistribute it and/or modify |  | ||||||
| # it under the terms of the GNU General Public License as published by |  | ||||||
| # the Free Software Foundation, either version 3 of the License, or |  | ||||||
| # (at your option) any later version. |  | ||||||
| # |  | ||||||
| # This program is distributed in the hope that it will be useful, |  | ||||||
| # but WITHOUT ANY WARRANTY; without even the implied warranty of |  | ||||||
| # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the |  | ||||||
| # GNU General Public License for more details. |  | ||||||
| # |  | ||||||
| # You should have received a copy of the GNU General Public License |  | ||||||
| # along with this program.  If not, see <http://www.gnu.org/licenses/>. |  | ||||||
| # |  | ||||||
| # --- |  | ||||||
| # |  | ||||||
| # Connects to a DSpace Solr statistics core and ingests views and downloads for | # Connects to a DSpace Solr statistics core and ingests views and downloads for | ||||||
| # communities, collections, and items into a PostgreSQL database. | # communities, collections, and items into a PostgreSQL database. | ||||||
| # | # | ||||||
| @@ -47,7 +31,7 @@ def index_views(indexType: str, facetField: str): | |||||||
|     # |     # | ||||||
|     # see: https://lucene.apache.org/solr/guide/6_6/the-stats-component.html |     # see: https://lucene.apache.org/solr/guide/6_6/the-stats-component.html | ||||||
|     solr_query_params = { |     solr_query_params = { | ||||||
|         "q": "type:2", |         "q": f"type:2 AND {facetField}:/.{{36}}/", | ||||||
|         "fq": "-isBot:true AND statistics_type:view", |         "fq": "-isBot:true AND statistics_type:view", | ||||||
|         "fl": facetField, |         "fl": facetField, | ||||||
|         "facet": "true", |         "facet": "true", | ||||||
| @@ -94,7 +78,7 @@ def index_views(indexType: str, facetField: str): | |||||||
|                 ) |                 ) | ||||||
|  |  | ||||||
|                 solr_query_params = { |                 solr_query_params = { | ||||||
|                     "q": "type:2", |                     "q": f"type:2 AND {facetField}:/.{{36}}/", | ||||||
|                     "fq": "-isBot:true AND statistics_type:view", |                     "fq": "-isBot:true AND statistics_type:view", | ||||||
|                     "fl": facetField, |                     "fl": facetField, | ||||||
|                     "facet": "true", |                     "facet": "true", | ||||||
| @@ -130,7 +114,7 @@ def index_views(indexType: str, facetField: str): | |||||||
| def index_downloads(indexType: str, facetField: str): | def index_downloads(indexType: str, facetField: str): | ||||||
|     # get the total number of distinct facets for items with at least 1 download |     # get the total number of distinct facets for items with at least 1 download | ||||||
|     solr_query_params = { |     solr_query_params = { | ||||||
|         "q": "type:0", |         "q": f"type:0 AND {facetField}:/.{{36}}/", | ||||||
|         "fq": "-isBot:true AND statistics_type:view AND bundleName:ORIGINAL", |         "fq": "-isBot:true AND statistics_type:view AND bundleName:ORIGINAL", | ||||||
|         "fl": facetField, |         "fl": facetField, | ||||||
|         "facet": "true", |         "facet": "true", | ||||||
| @@ -176,7 +160,7 @@ def index_downloads(indexType: str, facetField: str): | |||||||
|                 ) |                 ) | ||||||
|  |  | ||||||
|                 solr_query_params = { |                 solr_query_params = { | ||||||
|                     "q": "type:0", |                     "q": f"type:0 AND {facetField}:/.{{36}}/", | ||||||
|                     "fq": "-isBot:true AND statistics_type:view AND bundleName:ORIGINAL", |                     "fq": "-isBot:true AND statistics_type:view AND bundleName:ORIGINAL", | ||||||
|                     "fl": facetField, |                     "fl": facetField, | ||||||
|                     "facet": "true", |                     "facet": "true", | ||||||
|   | |||||||
| @@ -1,3 +1,5 @@ | |||||||
|  | # SPDX-License-Identifier: GPL-3.0-only | ||||||
|  |  | ||||||
| import requests | import requests | ||||||
|  |  | ||||||
| from .config import SOLR_SERVER | from .config import SOLR_SERVER | ||||||
|   | |||||||
| @@ -1,3 +1,5 @@ | |||||||
|  | # SPDX-License-Identifier: GPL-3.0-only | ||||||
|  |  | ||||||
| import datetime | import datetime | ||||||
| import json | import json | ||||||
| import re | import re | ||||||
|   | |||||||
							
								
								
									
										799
									
								
								poetry.lock
									
									
									
										generated
									
									
									
								
							
							
						
						
									
										799
									
								
								poetry.lock
									
									
									
										generated
									
									
									
								
							
										
											
												File diff suppressed because it is too large
												Load Diff
											
										
									
								
							| @@ -1,6 +1,6 @@ | |||||||
| [tool.poetry] | [tool.poetry] | ||||||
| name = "dspace-statistics-api" | name = "dspace-statistics-api" | ||||||
| version = "1.4.0" | version = "1.4.3-dev" | ||||||
| description = "A simple REST API to expose Solr view and download statistics for items, communities, and collections in a DSpace repository." | description = "A simple REST API to expose Solr view and download statistics for items, communities, and collections in a DSpace repository." | ||||||
| authors = ["Alan Orth <aorth@mjanja.ch>"] | authors = ["Alan Orth <aorth@mjanja.ch>"] | ||||||
| license = "GPL-3.0-only" | license = "GPL-3.0-only" | ||||||
| @@ -8,17 +8,17 @@ license = "GPL-3.0-only" | |||||||
| [tool.poetry.dependencies] | [tool.poetry.dependencies] | ||||||
| python = "^3.6" | python = "^3.6" | ||||||
| gunicorn = "^20.0.4" | gunicorn = "^20.0.4" | ||||||
| falcon = "^2.0.0" | falcon = "3.0.0" | ||||||
| psycopg2-binary = "^2.8.6" | psycopg2 = "^2.9.1" | ||||||
| requests = "^2.24.0" | requests = "^2.24.0" | ||||||
| falcon-swagger-ui = {git = "https://github.com/alanorth/falcon-swagger-ui.git"} | falcon-swagger-ui = {git = "https://github.com/rdidyk/falcon-swagger-ui.git", rev="818c09e17ae1231b935f18edb9823878dc6c7bfb"} | ||||||
|  |  | ||||||
| [tool.poetry.dev-dependencies] | [tool.poetry.dev-dependencies] | ||||||
| ipython = { version = "^7.18.1", python = "^3.7" } | ipython = { version = "^7.18.1", python = "^3.7" } | ||||||
| flake8 = "^3.8.4" | flake8 = "^3.8.4" | ||||||
| pytest = "^6.1.1" | pytest = "^6.1.1" | ||||||
| isort = "^5.5.4" | black = {version = "^21.6b0", python = ">=3.6.2"} | ||||||
| black = "^20.8b1" | isort = {version = "^5.9.1", python = ">=3.6.1"} | ||||||
|  |  | ||||||
| [build-system] | [build-system] | ||||||
| requires = ["poetry>=0.12"] | requires = ["poetry>=0.12"] | ||||||
|   | |||||||
| @@ -1,51 +1,52 @@ | |||||||
| appdirs==1.4.4 |  | ||||||
| appnope==0.1.2; python_version >= "3.7" and python_version < "4.0" and sys_platform == "darwin" | appnope==0.1.2; python_version >= "3.7" and python_version < "4.0" and sys_platform == "darwin" | ||||||
| atomicwrites==1.4.0; sys_platform == "win32" | atomicwrites==1.4.0; python_version >= "3.6" and python_full_version < "3.0.0" and sys_platform == "win32" or sys_platform == "win32" and python_version >= "3.6" and python_full_version >= "3.4.0" | ||||||
| attrs==20.3.0 | attrs==21.2.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version >= "3.6" | ||||||
| backcall==0.2.0; python_version >= "3.7" and python_version < "4.0" | backcall==0.2.0; python_version >= "3.7" and python_version < "4.0" | ||||||
| black==20.8b1 | black==21.9b0; python_version >= "3.6.2" | ||||||
| certifi==2020.12.5 | certifi==2021.10.8; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" | ||||||
| chardet==4.0.0 | charset-normalizer==2.0.7; python_full_version >= "3.6.0" and python_version >= "3" | ||||||
| click==7.1.2 | click==8.0.3; python_full_version >= "3.6.2" and python_version >= "3.6.2" | ||||||
| colorama==0.4.4; python_version >= "3.7" and python_version < "4.0" and sys_platform == "win32" or sys_platform == "win32" | colorama==0.4.4; python_version >= "3.7" and python_version < "4.0" and sys_platform == "win32" and python_full_version >= "3.6.2" and platform_system == "Windows" | ||||||
| dataclasses==0.6; python_version < "3.7" | dataclasses==0.8; python_version >= "3.6.2" and python_version < "3.7" and python_full_version >= "3.6.2" | ||||||
| decorator==4.4.2; python_version >= "3.7" and python_version < "4.0" | decorator==5.1.0; python_version >= "3.7" and python_version < "4.0" | ||||||
| falcon==2.0.0 | falcon-swagger-ui @ git+https://github.com/rdidyk/falcon-swagger-ui.git@818c09e17ae1231b935f18edb9823878dc6c7bfb | ||||||
| -e git+https://github.com/alanorth/falcon-swagger-ui.git@a44244c85dceccfcd249b62fea4ee82a8221e3d2#egg=falcon-swagger-ui | falcon==3.0.0; python_version >= "3.5" | ||||||
| flake8==3.8.4 | flake8==3.9.2; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") | ||||||
| gunicorn==20.0.4 | gunicorn==20.1.0; python_version >= "3.5" | ||||||
| idna==2.10 | idna==3.3; python_version >= "3.5" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.5" | ||||||
| importlib-metadata==3.3.0; python_version < "3.8" | importlib-metadata==4.8.1; python_full_version >= "3.6.2" and python_version < "3.8" and python_version >= "3.6.2" | ||||||
| iniconfig==1.1.1 | iniconfig==1.1.1; python_version >= "3.6" | ||||||
| ipython==7.19.0; python_version >= "3.7" and python_version < "4.0" | ipython==7.28.0; python_version >= "3.7" and python_version < "4.0" | ||||||
| ipython-genutils==0.2.0; python_version >= "3.7" and python_version < "4.0" | isort==5.9.3; python_version >= "3.6.1" | ||||||
| isort==5.6.4 | jedi==0.18.0; python_version >= "3.7" and python_version < "4.0" | ||||||
| jedi==0.17.2; python_version >= "3.7" and python_version < "4.0" | jinja2==3.0.2; python_version >= "3.6" | ||||||
| jinja2==2.11.2 | markupsafe==2.0.1; python_version >= "3.6" | ||||||
| markupsafe==1.1.1 | matplotlib-inline==0.1.3; python_version >= "3.7" and python_version < "4.0" | ||||||
| mccabe==0.6.1 | mccabe==0.6.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" | ||||||
| mypy-extensions==0.4.3 | mypy-extensions==0.4.3; python_full_version >= "3.6.2" and python_version >= "3.6.2" | ||||||
| packaging==20.8 | packaging==21.0; python_version >= "3.6" | ||||||
| parso==0.7.1; python_version >= "3.7" and python_version < "4.0" | parso==0.8.2; python_version >= "3.7" and python_version < "4.0" | ||||||
| pathspec==0.8.1 | pathspec==0.9.0; python_full_version >= "3.6.2" and python_version >= "3.6.2" | ||||||
| pexpect==4.8.0; python_version >= "3.7" and python_version < "4.0" and sys_platform != "win32" | pexpect==4.8.0; python_version >= "3.7" and python_version < "4.0" and sys_platform != "win32" | ||||||
| pickleshare==0.7.5; python_version >= "3.7" and python_version < "4.0" | pickleshare==0.7.5; python_version >= "3.7" and python_version < "4.0" | ||||||
| pluggy==0.13.1 | platformdirs==2.4.0; python_full_version >= "3.6.2" and python_version >= "3.6.2" | ||||||
| prompt-toolkit==3.0.8; python_version >= "3.7" and python_version < "4.0" | pluggy==1.0.0; python_version >= "3.6" | ||||||
| psycopg2-binary==2.8.6 | prompt-toolkit==3.0.20; python_version >= "3.7" and python_version < "4.0" and python_full_version >= "3.6.2" | ||||||
| ptyprocess==0.6.0; python_version >= "3.7" and python_version < "4.0" and sys_platform != "win32" | psycopg2==2.9.1; python_version >= "3.6" | ||||||
| py==1.10.0 | ptyprocess==0.7.0; python_version >= "3.7" and python_version < "4.0" and sys_platform != "win32" | ||||||
| pycodestyle==2.6.0 | py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" and python_version >= "3.6" | ||||||
| pyflakes==2.2.0 | pycodestyle==2.7.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" | ||||||
| pygments==2.7.3; python_version >= "3.7" and python_version < "4.0" | pyflakes==2.3.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" | ||||||
| pyparsing==2.4.7 | pygments==2.10.0; python_version >= "3.7" and python_version < "4.0" | ||||||
| pytest==6.2.1 | pyparsing==2.4.7; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" | ||||||
| regex==2020.11.13 | pytest==6.2.5; python_version >= "3.6" | ||||||
| requests==2.25.1 | regex==2021.10.21; python_full_version >= "3.6.2" and python_version >= "3.6.2" | ||||||
| toml==0.10.2 | requests==2.26.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0") | ||||||
| traitlets==5.0.5; python_version >= "3.7" and python_version < "4.0" | toml==0.10.2; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" and python_version >= "3.6" | ||||||
| typed-ast==1.4.1 | tomli==1.2.1; python_full_version >= "3.6.2" and python_version >= "3.6.2" | ||||||
| typing-extensions==3.7.4.3 | traitlets==5.1.0; python_version >= "3.7" and python_version < "4.0" | ||||||
| urllib3==1.26.2 | typed-ast==1.4.3; python_full_version >= "3.6.2" and python_version >= "3.6.2" and python_version < "3.8" | ||||||
| wcwidth==0.2.5; python_version >= "3.7" and python_version < "4.0" | typing-extensions==3.10.0.2 | ||||||
| zipp==3.4.0; python_version < "3.8" | urllib3==1.26.7; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "4" | ||||||
|  | wcwidth==0.2.5; python_version >= "3.7" and python_version < "4.0" and python_full_version >= "3.6.2" | ||||||
|  | zipp==3.6.0; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "3.8" or python_full_version >= "3.5.0" and python_version < "3.8" and python_version >= "3.6" | ||||||
|   | |||||||
| @@ -1,11 +1,11 @@ | |||||||
| certifi==2020.12.5 | certifi==2021.10.8; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" | ||||||
| chardet==4.0.0 | charset-normalizer==2.0.7; python_full_version >= "3.6.0" and python_version >= "3" | ||||||
| falcon==2.0.0 | falcon-swagger-ui @ git+https://github.com/rdidyk/falcon-swagger-ui.git@818c09e17ae1231b935f18edb9823878dc6c7bfb | ||||||
| -e git+https://github.com/alanorth/falcon-swagger-ui.git@a44244c85dceccfcd249b62fea4ee82a8221e3d2#egg=falcon-swagger-ui | falcon==3.0.0; python_version >= "3.5" | ||||||
| gunicorn==20.0.4 | gunicorn==20.1.0; python_version >= "3.5" | ||||||
| idna==2.10 | idna==3.3; python_version >= "3.5" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.5" | ||||||
| jinja2==2.11.2 | jinja2==3.0.2; python_version >= "3.6" | ||||||
| markupsafe==1.1.1 | markupsafe==2.0.1; python_version >= "3.6" | ||||||
| psycopg2-binary==2.8.6 | psycopg2==2.9.1; python_version >= "3.6" | ||||||
| requests==2.25.1 | requests==2.26.0; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.6.0") | ||||||
| urllib3==1.26.2 | urllib3==1.26.7; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version < "4" | ||||||
|   | |||||||
| @@ -1,14 +1,17 @@ | |||||||
| from falcon import testing | # SPDX-License-Identifier: GPL-3.0-only | ||||||
|  |  | ||||||
| import json | import json | ||||||
| import pytest |  | ||||||
| from unittest.mock import patch | from unittest.mock import patch | ||||||
|  |  | ||||||
| from dspace_statistics_api.app import api | import pytest | ||||||
|  | from falcon import testing | ||||||
|  |  | ||||||
|  | from dspace_statistics_api.app import app | ||||||
|  |  | ||||||
|  |  | ||||||
| @pytest.fixture | @pytest.fixture | ||||||
| def client(): | def client(): | ||||||
|     return testing.TestClient(api) |     return testing.TestClient(app) | ||||||
|  |  | ||||||
|  |  | ||||||
| def test_get_collection(client): | def test_get_collection(client): | ||||||
| @@ -309,7 +312,7 @@ def test_post_collections_valid_page(client): | |||||||
|     assert response.status_code == 200 |     assert response.status_code == 200 | ||||||
|     assert response.json["limit"] == 100 |     assert response.json["limit"] == 100 | ||||||
|     assert response.json["currentPage"] == 0 |     assert response.json["currentPage"] == 0 | ||||||
|     assert response.json["totalPages"] == 0 |     assert response.json["totalPages"] == 1 | ||||||
|     assert len(response.json["statistics"]) == 2 |     assert len(response.json["statistics"]) == 2 | ||||||
|     assert isinstance(response.json["statistics"][0]["views"], int) |     assert isinstance(response.json["statistics"][0]["views"], int) | ||||||
|     assert isinstance(response.json["statistics"][0]["downloads"], int) |     assert isinstance(response.json["statistics"][0]["downloads"], int) | ||||||
|   | |||||||
| @@ -1,14 +1,17 @@ | |||||||
| from falcon import testing | # SPDX-License-Identifier: GPL-3.0-only | ||||||
|  |  | ||||||
| import json | import json | ||||||
| import pytest |  | ||||||
| from unittest.mock import patch | from unittest.mock import patch | ||||||
|  |  | ||||||
| from dspace_statistics_api.app import api | import pytest | ||||||
|  | from falcon import testing | ||||||
|  |  | ||||||
|  | from dspace_statistics_api.app import app | ||||||
|  |  | ||||||
|  |  | ||||||
| @pytest.fixture | @pytest.fixture | ||||||
| def client(): | def client(): | ||||||
|     return testing.TestClient(api) |     return testing.TestClient(app) | ||||||
|  |  | ||||||
|  |  | ||||||
| def test_get_community(client): | def test_get_community(client): | ||||||
| @@ -309,7 +312,7 @@ def test_post_communities_valid_page(client): | |||||||
|     assert response.status_code == 200 |     assert response.status_code == 200 | ||||||
|     assert response.json["limit"] == 100 |     assert response.json["limit"] == 100 | ||||||
|     assert response.json["currentPage"] == 0 |     assert response.json["currentPage"] == 0 | ||||||
|     assert response.json["totalPages"] == 0 |     assert response.json["totalPages"] == 1 | ||||||
|     assert len(response.json["statistics"]) == 2 |     assert len(response.json["statistics"]) == 2 | ||||||
|     assert isinstance(response.json["statistics"][0]["views"], int) |     assert isinstance(response.json["statistics"][0]["views"], int) | ||||||
|     assert isinstance(response.json["statistics"][0]["downloads"], int) |     assert isinstance(response.json["statistics"][0]["downloads"], int) | ||||||
|   | |||||||
| @@ -1,12 +1,14 @@ | |||||||
| from falcon import testing | # SPDX-License-Identifier: GPL-3.0-only | ||||||
| import pytest |  | ||||||
|  |  | ||||||
| from dspace_statistics_api.app import api | import pytest | ||||||
|  | from falcon import testing | ||||||
|  |  | ||||||
|  | from dspace_statistics_api.app import app | ||||||
|  |  | ||||||
|  |  | ||||||
| @pytest.fixture | @pytest.fixture | ||||||
| def client(): | def client(): | ||||||
|     return testing.TestClient(api) |     return testing.TestClient(app) | ||||||
|  |  | ||||||
|  |  | ||||||
| def test_get_docs(client): | def test_get_docs(client): | ||||||
|   | |||||||
| @@ -1,14 +1,17 @@ | |||||||
| from falcon import testing | # SPDX-License-Identifier: GPL-3.0-only | ||||||
|  |  | ||||||
| import json | import json | ||||||
| import pytest |  | ||||||
| from unittest.mock import patch | from unittest.mock import patch | ||||||
|  |  | ||||||
| from dspace_statistics_api.app import api | import pytest | ||||||
|  | from falcon import testing | ||||||
|  |  | ||||||
|  | from dspace_statistics_api.app import app | ||||||
|  |  | ||||||
|  |  | ||||||
| @pytest.fixture | @pytest.fixture | ||||||
| def client(): | def client(): | ||||||
|     return testing.TestClient(api) |     return testing.TestClient(app) | ||||||
|  |  | ||||||
|  |  | ||||||
| def test_get_item(client): | def test_get_item(client): | ||||||
| @@ -309,7 +312,7 @@ def test_post_items_valid_page(client): | |||||||
|     assert response.status_code == 200 |     assert response.status_code == 200 | ||||||
|     assert response.json["limit"] == 100 |     assert response.json["limit"] == 100 | ||||||
|     assert response.json["currentPage"] == 0 |     assert response.json["currentPage"] == 0 | ||||||
|     assert response.json["totalPages"] == 0 |     assert response.json["totalPages"] == 1 | ||||||
|     assert len(response.json["statistics"]) == 2 |     assert len(response.json["statistics"]) == 2 | ||||||
|     assert isinstance(response.json["statistics"][0]["views"], int) |     assert isinstance(response.json["statistics"][0]["views"], int) | ||||||
|     assert isinstance(response.json["statistics"][0]["downloads"], int) |     assert isinstance(response.json["statistics"][0]["downloads"], int) | ||||||
|   | |||||||
		Reference in New Issue
	
	Block a user