CHANGELOG.md: Move unrelease changes to 0.9.0

CHANGELOG.md: Add note about catching errors
dspace_statistics_api/indexer.py: Catch case of no views/downloads
2025-05-10 15:16:02 +02:00 · 2019-01-22 09:02:29 +02:00 · 2019-01-22 09:01:54 +02:00 · 2019-01-22 09:00:22 +02:00 · 2019-01-22 08:41:39 +02:00 · 2019-01-22 08:41:31 +02:00
7 changed files with 190 additions and 142 deletions
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@ -4,6 +4,20 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

+## [0.9.0] - 2019-01-22
+### Updated
+- pytest version 4.0.0
+- Fix indexing of sharded statistics cores ([#10))
+- Handle case of missing views/downloads gracefully
+
+## [0.8.1] - 2018-11-14
+### Changed
+- README.md to recommend using vanilla Python virtual environments and pip instead of pipenv
+- Regenerate pipenv environment to capture only direct dependencies
+
+### Added
+- `requirements-dev.txt` for installing development packages with pip
+
 ## [0.8.0] - 2018-11-11
 ### Changed
 - Properly handle database connection errors
--- a/18
+++ b/18
@ -4,23 +4,15 @@ verify_ssl = true
 name = "pypi"

 [packages]
-certifi = "==2018.10.15"
-chardet = "==3.0.4"
-falcon = "==1.4.1"
-gunicorn = "==19.9.0"
-idna = "==2.7"
-kazoo = "==2.5.0"
-"psycopg2-binary" = "==2.7.5"
-python-mimeparse = "==1.6.0"
-requests = "==2.20.0"
-six = "==1.11.0"
+gunicorn = "*"
+falcon = "*"
+"psycopg2-binary" = "*"
 solrclient = {ref = "kazoo-2.5.0", git = "https://github.com/alanorth/SolrClient.git"}
-"urllib3" = "==1.24"

 [dev-packages]
-"flake8" = "*"
 ipython = "*"
+"flake8" = "*"
 pytest = "*"

 [requires]
-python_version = "3.6"
+python_version = "3.7"
--- a/Pipfile.lock
+++ b/Pipfile.lock
@ -1,11 +1,11 @@
 {
    "_meta": {
        "hash": {
-            "sha256": "da754a9f6c668303b071155fdd3a31067897f2c1703a28990745506a5ea55538"
+            "sha256": "a846fdab4de5765a7e7fc19424a97a6196248e29f87285cf81fd76e8e9ae3e28"
        },
        "pipfile-spec": 6,
        "requires": {
-            "python_version": "3.6"
+            "python_version": "3.7"
        },
        "sources": [
            {
@ -16,22 +16,6 @@
        ]
    },
    "default": {
-        "certifi": {
-            "hashes": [
-                "sha256:339dc09518b07e2fa7eda5450740925974815557727d6bd35d319c1524a04a4c",
-                "sha256:6d58c986d22b038c8c0df30d639f23a3e6d172a05c3583e766f4c0b785c0986a"
-            ],
-            "index": "pypi",
-            "version": "==2018.10.15"
-        },
-        "chardet": {
-            "hashes": [
-                "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae",
-                "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"
-            ],
-            "index": "pypi",
-            "version": "==3.0.4"
-        },
        "falcon": {
            "hashes": [
                "sha256:0a66b33458fab9c1e400a9be1a68056abda178eb02a8cb4b8f795e9df20b053b",
@ -48,93 +32,59 @@
            "index": "pypi",
            "version": "==19.9.0"
        },
-        "idna": {
-            "hashes": [
-                "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e",
-                "sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16"
-            ],
-            "index": "pypi",
-            "version": "==2.7"
-        },
-        "kazoo": {
-            "hashes": [
-                "sha256:8db774f7bdece7d0dc7decb21539ff0852e42c2ffe1c28d7f1ff6f9292a1c3a4",
-                "sha256:a5fa2e400c5068cfee9e86b35cf0dab8232b574152d8e3590d823b3e2426ab5e"
-            ],
-            "index": "pypi",
-            "version": "==2.5.0"
-        },
        "psycopg2-binary": {
            "hashes": [
-                "sha256:04afb59bbbd2eab3148e6816beddc74348078b8c02a1113ea7f7822f5be4afe3",
-                "sha256:098b18f4d8857a8f9b206d1dc54db56c2255d5d26458917e7bcad61ebfe4338f",
-                "sha256:0bf855d4a7083e20ead961fda4923887094eaeace0ab2d76eb4aa300f4bbf5bd",
-                "sha256:197dda3ffd02057820be83fe4d84529ea70bf39a9a4daee1d20ffc74eb3d042e",
-                "sha256:278ef63afb4b3d842b4609f2c05ffbfb76795cf6a184deeb8707cd5ed3c981a5",
-                "sha256:3cbf8c4fc8f22f0817220891cf405831559f4d4c12c4f73913730a2ea6c47a47",
-                "sha256:4305aed922c4d9d6163ab3a41d80b5a1cfab54917467da8168552c42cad84d32",
-                "sha256:47ee296f704fb8b2a616dec691cdcfd5fa0f11943955e88faa98cbd1dc3b3e3d",
-                "sha256:4a0e38cb30457e70580903367161173d4a7d1381eb2f2cfe4e69b7806623f484",
-                "sha256:4d6c294c6638a71cafb82a37f182f24321f1163b08b5d5ca076e11fe838a3086",
-                "sha256:4f3233c366500730f839f92833194fd8f9a5c4529c8cd8040aa162c3740de8e5",
-                "sha256:5221f5a3f4ca2ddf0d58e8b8a32ca50948be9a43351fda797eb4e72d7a7aa34d",
-                "sha256:5c6ca0b507540a11eaf9e77dee4f07c131c2ec80ca0cffa146671bf690bc1c02",
-                "sha256:789bd89d71d704db2b3d5e67d6d518b158985d791d3b2dec5ab85457cfc9677b",
-                "sha256:7b94d29239efeaa6a967f3b5971bd0518d2a24edd1511edbf4a2c8b815220d07",
-                "sha256:89bc65ef3301c74cf32db25334421ea6adbe8f65601ea45dcaaf095abed910bb",
-                "sha256:89d6d3a549f405c20c9ae4dc94d7ed2de2fa77427a470674490a622070732e62",
-                "sha256:97521704ac7127d7d8ba22877da3c7bf4a40366587d238ec679ff38e33177498",
-                "sha256:a395b62d5f44ff6f633231abe568e2203b8fabf9797cd6386aa92497df912d9a",
-                "sha256:a6d32c37f714c3f34158f3fa659f3a8f2658d5f53c4297d45579b9677cc4d852",
-                "sha256:a89ee5c26f72f2d0d74b991ce49e42ddeb4ac0dc2d8c06a0f2770a1ab48f4fe0",
-                "sha256:b4c8b0ef3608e59317bfc501df84a61e48b5445d45f24d0391a24802de5f2d84",
-                "sha256:b5fcf07140219a1f71e18486b8dc28e2e1b76a441c19374805c617aa6d9a9d55",
-                "sha256:b86f527f00956ecebad6ab3bb30e3a75fedf1160a8716978dd8ce7adddedd86f",
-                "sha256:be4c4aa22ba22f70de36c98b06480e2f1697972d49eb20d525f400d204a6d272",
-                "sha256:c2ac7aa1a144d4e0e613ac7286dae85671e99fe7a1353954d4905629c36b811c",
-                "sha256:de26ef4787b5e778e8223913a3e50368b44e7480f83c76df1f51d23bd21cea16",
-                "sha256:e70ebcfc5372dc7b699c0110454fc4263967f30c55454397e5769eb72c0eb0ce",
-                "sha256:eadbd32b6bc48b67b0457fccc94c86f7ccc8178ab839f684eb285bb592dc143e",
-                "sha256:ecbc6dfff6db06b8b72ae8a2f25ff20fbdcb83cb543811a08f7cb555042aa729"
+                "sha256:036bcb198a7cc4ce0fe43344f8c2c9a8155aefa411633f426c8c6ed58a6c0426",
+                "sha256:1d770fcc02cdf628aebac7404d56b28a7e9ebec8cfc0e63260bd54d6edfa16d4",
+                "sha256:1fdc6f369dcf229de6c873522d54336af598b9470ccd5300e2f58ee506f5ca13",
+                "sha256:21f9ddc0ff6e07f7d7b6b484eb9da2c03bc9931dd13e36796b111d631f7135a3",
+                "sha256:247873cda726f7956f745a3e03158b00de79c4abea8776dc2f611d5ba368d72d",
+                "sha256:3aa31c42f29f1da6f4fd41433ad15052d5ff045f2214002e027a321f79d64e2c",
+                "sha256:475f694f87dbc619010b26de7d0fc575a4accf503f2200885cc21f526bffe2ad",
+                "sha256:4b5e332a24bf6e2fda1f51ca2a57ae1083352293a08eeea1fa1112dc7dd542d1",
+                "sha256:570d521660574aca40be7b4d532dfb6f156aad7b16b5ed62d1534f64f1ef72d8",
+                "sha256:59072de7def0690dd13112d2bdb453e20570a97297070f876fbbb7cbc1c26b05",
+                "sha256:5f0b658989e918ef187f8a08db0420528126f2c7da182a7b9f8bf7f85144d4e4",
+                "sha256:649199c84a966917d86cdc2046e03d536763576c0b2a756059ae0b3a9656bc20",
+                "sha256:6645fc9b4705ae8fbf1ef7674f416f89ae1559deec810f6dd15197dfa52893da",
+                "sha256:6872dd54d4e398d781efe8fe2e2d7eafe4450d61b5c4898aced7610109a6df75",
+                "sha256:6ce34fbc251fc0d691c8d131250ba6f42fd2b28ef28558d528ba8c558cb28804",
+                "sha256:73920d167a0a4d1006f5f3b9a3efce6f0e5e883a99599d38206d43f27697df00",
+                "sha256:8a671732b87ae423e34b51139628123bc0306c2cb85c226e71b28d3d57d7e42a",
+                "sha256:8d517e8fda2efebca27c2018e14c90ed7dc3f04d7098b3da2912e62a1a5585fe",
+                "sha256:9475a008eb7279e20d400c76471843c321b46acacc7ee3de0b47233a1e3fa2cf",
+                "sha256:96947b8cd7b3148fb0e6549fcb31258a736595d6f2a599f8cd450e9a80a14781",
+                "sha256:abf229f24daa93f67ac53e2e17c8798a71a01711eb9fcdd029abba8637164338",
+                "sha256:b1ab012f276df584beb74f81acb63905762c25803ece647016613c3d6ad4e432",
+                "sha256:b22b33f6f0071fe57cb4e9158f353c88d41e739a3ec0d76f7b704539e7076427",
+                "sha256:b3b2d53274858e50ad2ffdd6d97ce1d014e1e530f82ec8b307edd5d4c921badf",
+                "sha256:bab26a729befc7b9fab9ded1bba9c51b785188b79f8a2796ba03e7e734269e2e",
+                "sha256:daa1a593629aa49f506eddc9d23dc7f89b35693b90e1fbcd4480182d1203ea90",
+                "sha256:dd111280ce40e89fd17b19c1269fd1b74a30fce9d44a550840e86edb33924eb8",
+                "sha256:e0b86084f1e2e78c451994410de756deba206884d6bed68d5a3d7f39ff5fea1d",
+                "sha256:eb86520753560a7e89639500e2a254bb6f683342af598088cb72c73edcad21e6",
+                "sha256:ff18c5c40a38d41811c23e2480615425c97ea81fd7e9118b8b899c512d97c737"
            ],
            "index": "pypi",
-            "version": "==2.7.5"
+            "version": "==2.7.6.1"
        },
        "python-mimeparse": {
            "hashes": [
                "sha256:76e4b03d700a641fd7761d3cd4fdbbdcd787eade1ebfac43f877016328334f78",
                "sha256:a295f03ff20341491bfe4717a39cd0a8cc9afad619ba44b77e86b0ab8a2b8282"
            ],
-            "index": "pypi",
            "version": "==1.6.0"
        },
-        "requests": {
-            "hashes": [
-                "sha256:99dcfdaaeb17caf6e526f32b6a7b780461512ab3f1d992187801694cba42770c",
-                "sha256:a84b8c9ab6239b578f22d1c21d51b696dcfe004032bb80ea832398d6909d7279"
-            ],
-            "index": "pypi",
-            "version": "==2.20.0"
-        },
        "six": {
            "hashes": [
-                "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9",
-                "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb"
+                "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
+                "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"
            ],
-            "index": "pypi",
-            "version": "==1.11.0"
+            "version": "==1.12.0"
        },
        "solrclient": {
            "git": "https://github.com/alanorth/SolrClient.git",
            "ref": "c629e3475be37c82770b2be61748be7e29882648"
-        },
-        "urllib3": {
-            "hashes": [
-                "sha256:41c3db2fc01e5b907288010dec72f9d0a74e37d6994e6eb56849f59fea2265ae",
-                "sha256:8819bba37a02d143296a4d032373c4dd4aca11f6d4c9973335ca75f9c8475f59"
-            ],
-            "index": "pypi",
-            "version": "==1.24"
        }
    },
    "develop": {
@ -176,11 +126,11 @@
        },
        "ipython": {
            "hashes": [
-                "sha256:a5781d6934a3341a1f9acb4ea5acdc7ea0a0855e689dbe755d070ca51e995435",
-                "sha256:b10a7ddd03657c761fc503495bc36471c8158e3fc948573fb9fe82a7029d8efd"
+                "sha256:6a9496209b76463f1dec126ab928919aaf1f55b38beb9219af3fe202f6bbdd12",
+                "sha256:f69932b1e806b38a7818d9a1e918e5821b685715040b48e59c657b3c7961b742"
            ],
            "index": "pypi",
-            "version": "==7.1.1"
+            "version": "==7.2.0"
        },
        "ipython-genutils": {
            "hashes": [
@ -191,10 +141,10 @@
        },
        "jedi": {
            "hashes": [
-                "sha256:0191c447165f798e6a730285f2eee783fff81b0d3df261945ecb80983b5c3ca7",
-                "sha256:b7493f73a2febe0dc33d51c99b474547f7f6c0b2c8fb2b21f453eef204c12148"
+                "sha256:571702b5bd167911fe9036e5039ba67f820d6502832285cde8c881ab2b2149fd",
+                "sha256:c8481b5e59d34a5c7c42e98f6625e633f6ef59353abea6437472c7ec2093f191"
            ],
-            "version": "==0.13.1"
+            "version": "==0.13.2"
        },
        "mccabe": {
            "hashes": [
@ -205,11 +155,11 @@
        },
        "more-itertools": {
            "hashes": [
-                "sha256:c187a73da93e7a8acc0001572aebc7e3c69daf7bf6881a2cea10650bd4420092",
-                "sha256:c476b5d3a34e12d40130bc2f935028b5f636df8f372dc2c1c01dc19681b2039e",
-                "sha256:fcbfeaea0be121980e15bc97b3817b5202ca73d0eae185b4550cbfce2a3ebb3d"
+                "sha256:38a936c0a6d98a38bcc2d03fdaaedaba9f412879461dd2ceff8d37564d6522e4",
+                "sha256:c0a5785b1109a6bd7fac76d6837fd1feca158e54e521ccd2ae8bfe393cc9d4fc",
+                "sha256:fe7a7cae1ccb57d33952113ff4fa1bc5f879963600ed74918f1236e212ee50b9"
            ],
-            "version": "==4.3.0"
+            "version": "==5.0.0"
        },
        "parso": {
            "hashes": [
@ -235,10 +185,10 @@
        },
        "pluggy": {
            "hashes": [
-                "sha256:447ba94990e8014ee25ec853339faf7b0fc8050cdc3289d4d71f7f410fb90095",
-                "sha256:bde19360a8ec4dfd8a20dcb811780a30998101f078fc7ded6162f0076f50508f"
+                "sha256:8ddc32f03971bfdf900a81961a48ccf2fb677cf7715108f85295c67405798616",
+                "sha256:980710797ff6a041e9a73a5787804f848996ecaa6f8a1b1e08224a5894f2074a"
            ],
-            "version": "==0.8.0"
+            "version": "==0.8.1"
        },
        "prompt-toolkit": {
            "hashes": [
@ -278,26 +228,25 @@
        },
        "pygments": {
            "hashes": [
-                "sha256:78f3f434bcc5d6ee09020f92ba487f95ba50f1e3ef83ae96b9d5ffa1bab25c5d",
-                "sha256:dbae1046def0efb574852fab9e90209b23f556367b5a320c0bcb871c77c3e8cc"
+                "sha256:5ffada19f6203563680669ee7f53b64dabbeb100eb51b61996085e99c03b284a",
+                "sha256:e8218dd399a61674745138520d0d4cf2621d7e032439341bc3f647bff125818d"
            ],
-            "version": "==2.2.0"
+            "version": "==2.3.1"
        },
        "pytest": {
            "hashes": [
-                "sha256:630ff1dbe04f469ee78faa5660f712e58b953da7df22ea5d828c9012e134da43",
-                "sha256:a2b5232735dd0b736cbea9c0f09e5070d78fcaba2823a4f6f09d9a81bd19415c"
+                "sha256:41568ea7ecb4a68d7f63837cf65b92ce8d0105e43196ff2b26622995bb3dc4b2",
+                "sha256:c3c573a29d7c9547fb90217ece8a8843aa0c1328a797e200290dc3d0b4b823be"
            ],
            "index": "pypi",
-            "version": "==3.10.0"
+            "version": "==4.1.1"
        },
        "six": {
            "hashes": [
-                "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9",
-                "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb"
+                "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
+                "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"
            ],
-            "index": "pypi",
-            "version": "==1.11.0"
+            "version": "==1.12.0"
        },
        "traitlets": {
            "hashes": [
--- a/README.md
+++ b/README.md
@ -9,11 +9,14 @@ This project contains an indexer and a [Falcon-based](https://falcon.readthedocs
 - PostgreSQL version 9.5+ (due to [`UPSERT` support](https://wiki.postgresql.org/wiki/UPSERT))
 - DSpace with [Solr usage statistics enabled](https://wiki.duraspace.org/display/DSDOC5x/SOLR+Statistics) (tested with 5.x)

-## Installation and Testing
-Create a Python virtual environment and install the dependencies using [`pipenv`](https://github.com/pypa/pipenv):
+## Installation
+Create a Python virtual environment and install the dependencies:

-    $ pipenv install --dev
-    $ pipenv shell
+    $ python3 -m venv venv
+    $ source venv/bin/activate
+    $ pip install -r requirements.txt
+
+## Running

 Set up the environment variables for Solr and PostgreSQL:

@ -35,6 +38,11 @@ Test to see if there are any statistics:

    $ curl 'http://localhost:8000/items?limit=1'

+## Testing
+Install development packages using pip:
+
+    $ pip install -r requirements-dev.txt
+
 Run tests:

    $ pytest
@ -77,7 +85,10 @@ The item id is the *internal* id for an item. You can get these from the standar
 - Better logging
 - Version API
 - Use JSON in PostgreSQL
+- Make community and collection stats available
 - Switch to [Python 3.6+ f-string syntax](https://realpython.com/python-f-strings/)

 ## License
 This work is licensed under the [GPLv3](https://www.gnu.org/licenses/gpl-3.0.en.html).
+
+The license allows you to use and modify the work for personal and commercial purposes, but if you distribute the work you must provide users with a means to access the source code for the version you are distributing. Read more about the [GPLv3 at TL;DR Legal](https://tldrlegal.com/license/gnu-general-public-license-v3-(gpl-3)).
--- a/dspace_statistics_api/indexer.py
+++ b/dspace_statistics_api/indexer.py
@ -32,9 +32,56 @@
 from .database import DatabaseManager
 import json
 import psycopg2.extras
+import re
+import requests
 from .solr import solr_connection


+# Enumerate the cores in Solr to determine if statistics have been sharded into
+# yearly shards by DSpace's stats-util or not (for example: statistics-2018).
+def get_statistics_shards():
+    # Initialize an empty list for statistics core years
+    statistics_core_years = []
+
+    # URL for Solr status to check active cores
+    solr_url = solr.host + '/admin/cores?action=STATUS&wt=json'
+    res = requests.get(solr_url)
+
+    if res.status_code == requests.codes.ok:
+        data = res.json()
+
+        # Iterate over active cores from Solr's STATUS response (cores are in
+        # the status array of this response).
+        for core in data['status']:
+            # Pattern to match, for example: statistics-2018
+            pattern = re.compile('^statistics-[0-9]{4}$')
+
+            if not pattern.match(core):
+                continue
+
+            # Append current core to list
+            statistics_core_years.append(core)
+
+    # Initialize a string to hold our shards (may end up being empty if the Solr
+    # core has not been processed by stats-util).
+    shards = str()
+
+    if len(statistics_core_years) > 0:
+        # Begin building a string of shards starting with the default one
+        shards = '{}/statistics'.format(solr.host)
+
+        for core in statistics_core_years:
+            # Create a comma-separated list of shards to pass to our Solr query
+            #
+            # See: https://wiki.apache.org/solr/DistributedSearch
+            shards += ',{}/{}'.format(solr.host, core)
+
+    # Return the string of shards, which may actually be empty. Solr doesn't
+    # seem to mind if the shards query parameter is empty and I haven't seen
+    # any negative performance impact so this should be fine.
+    return shards
+
+
 def index_views():
    # get total number of distinct facets for items with a minimum of 1 view,
    # otherwise Solr returns all kinds of weird ids that are actually not in
@ -52,11 +99,17 @@ def index_views():
        'facet.offset': 0,
        'stats': True,
        'stats.field': 'id',
-        'stats.calcdistinct': True
+        'stats.calcdistinct': True,
+        'shards': shards
    }, rows=0)

-    # get total number of distinct facets (countDistinct)
-    results_totalNumFacets = json.loads(res.get_json())['stats']['stats_fields']['id']['countDistinct']
+    try:
+        # get total number of distinct facets (countDistinct)
+        results_totalNumFacets = json.loads(res.get_json())['stats']['stats_fields']['id']['countDistinct']
+    except TypeError:
+        print('No item views to index, exiting.')
+
+        exit(0)

    # divide results into "pages" (cast to int to effectively round down)
    results_per_page = 100
@ -78,7 +131,8 @@ def index_views():
                    'facet.field': 'id',
                    'facet.mincount': 1,
                    'facet.limit': results_per_page,
-                    'facet.offset': results_current_page * results_per_page
+                    'facet.offset': results_current_page * results_per_page,
+                    'shards': shards
                }, rows=0)

                # SolrClient's get_facets() returns a dict of dicts
@ -110,11 +164,17 @@ def index_downloads():
        'facet.offset': 0,
        'stats': True,
        'stats.field': 'owningItem',
-        'stats.calcdistinct': True
+        'stats.calcdistinct': True,
+        'shards': shards
    }, rows=0)

-    # get total number of distinct facets (countDistinct)
-    results_totalNumFacets = json.loads(res.get_json())['stats']['stats_fields']['owningItem']['countDistinct']
+    try:
+        # get total number of distinct facets (countDistinct)
+        results_totalNumFacets = json.loads(res.get_json())['stats']['stats_fields']['owningItem']['countDistinct']
+    except TypeError:
+        print('No item downloads to index, exiting.')
+
+        exit(0)

    # divide results into "pages" (cast to int to effectively round down)
    results_per_page = 100
@ -136,7 +196,8 @@ def index_downloads():
                    'facet.field': 'owningItem',
                    'facet.mincount': 1,
                    'facet.limit': results_per_page,
-                    'facet.offset': results_current_page * results_per_page
+                    'facet.offset': results_current_page * results_per_page,
+                    'shards': shards
                }, rows=0)

                # SolrClient's get_facets() returns a dict of dicts
@ -167,6 +228,8 @@ with DatabaseManager() as db:
    # commit the table creation before closing the database connection
    db.commit()

+shards = get_statistics_shards()
+
 index_views()
 index_downloads()

--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@ -0,0 +1,25 @@
+-i https://pypi.org/simple
+atomicwrites==1.2.1
+attrs==18.2.0
+backcall==0.1.0
+decorator==4.3.0
+flake8==3.6.0
+ipython-genutils==0.2.0
+ipython==7.2.0
+jedi==0.13.2
+mccabe==0.6.1
+more-itertools==5.0.0
+parso==0.3.1
+pexpect==4.6.0 ; sys_platform != 'win32'
+pickleshare==0.7.5
+pluggy==0.8.1
+prompt-toolkit==2.0.7
+ptyprocess==0.6.0
+py==1.7.0
+pycodestyle==2.4.0
+pyflakes==2.0.0
+pygments==2.3.1
+pytest==4.1.1
+six==1.12.0
+traitlets==4.3.2
+wcwidth==0.1.7
--- a/requirements.txt
+++ b/requirements.txt
@ -1,13 +1,7 @@
 -i https://pypi.org/simple
-certifi==2018.10.15
-chardet==3.0.4
 falcon==1.4.1
 git+https://github.com/alanorth/SolrClient.git@c629e3475be37c82770b2be61748be7e29882648#egg=solrclient
 gunicorn==19.9.0
-idna==2.7
-kazoo==2.5.0
-psycopg2-binary==2.7.5
+psycopg2-binary==2.7.6.1
 python-mimeparse==1.6.0
-requests==2.20.0
-six==1.11.0
-urllib3==1.24
+six==1.12.0
Author	SHA1	Message	Date
Alan Orth	914ec52fbb	CHANGELOG.md: Move unrelease changes to 0.9.0	2019-01-22 09:02:29 +02:00
Alan Orth	5524066656	CHANGELOG.md: Add note about catching errors	2019-01-22 09:01:54 +02:00
Alan Orth	043d897cef	dspace_statistics_api/indexer.py: Catch case of no views/downloads Don't fail with an exception when there are no views or downloads, for example on a new DSpace installation.	2019-01-22 09:00:22 +02:00
Alan Orth	bd28353cda	README.md: Remove TODO for fixing querying of shards	2019-01-22 08:41:39 +02:00
Alan Orth	e23d66c2a2	CHANGELOG.md: Add note about fixing querying of sharded cores	2019-01-22 08:41:31 +02:00
Alan Orth	40e284dac0	dspace_statistics_api/indexer.py: Query multiple shards DSpace's stats-util script splits the Solr statistics core into yearly shards. We need to use Solr's `shards` query parameter in order to get the statistics for previous years. This commit adds a helper function to enumerate the active Solr cores to find yearly shards matching the statistics-YYYY pattern and add them to the query.	2019-01-22 08:39:36 +02:00
Alan Orth	934fa9db9b	README.md: Add TODO about sharded statistics cores	2019-01-21 12:55:43 +02:00
Alan Orth	1fabb72b58	Update requirements Generated from pipenv with: $ pipenv lock -r > requirements.txt $ pipenv lock -r -d > requirements-dev.txt	2019-01-16 12:34:50 +02:00
Alan Orth	c7f95f0b60	README.md: Update TODO I think it might be possible to compute community and collection statistics from Solr and make them available at new endpoints: - /communities - /community/id - /collections - /collection/id	2019-01-16 09:59:29 +02:00
Alan Orth	c95a98dd2d	Pipfile.lock: update dependencies Updated with `pipenv update`.	2019-01-15 10:22:46 +02:00
Alan Orth	3f70f94a10	Pipfile.lock: Run pipenv update	2018-11-26 11:53:37 +02:00
Alan Orth	9b8ad9defd	Merge pull request #9 from ilri/pipenv-update Pipenv update	2018-11-19 23:50:44 +02:00
Alan Orth	d69ab20220	CHANGELOG.md: pytest version 4.0.0	2018-11-19 23:46:03 +02:00
Alan Orth	378f56ddc2	Pipfile.lock: Run pipenv update	2018-11-19 23:34:34 +02:00
Alan Orth	5a2a7d684c	CHANGELOG.md: Move unreleased changes to version 0.8.1	2018-11-14 09:37:00 +02:00
Alan Orth	18276e910f	CHANGELOG.md: Add notes about pipenv	2018-11-14 09:36:13 +02:00
Alan Orth	8de8c2765f	Merge pull request #8 from ilri/update-dependencies Update dependencies	2018-11-14 09:34:45 +02:00
Alan Orth	11a1755e59	Update requirements.txt Generated from pipenv with: $ pipenv lock -r > requirements.txt	2018-11-14 09:19:47 +02:00
Alan Orth	a835b0fdc5	Re-create pipenv environment from scratch When I originally created the pipenv environment I used the standard pip requirements.txt that I already had, which captured all the mod- ules and their exact versions at the time. This makes it hard to se- parate the project's actual dependencies from the dependencies' dep- endencies, complicating the Pipfile and making it hard to update mo- dule versions later. I've re-created the environment with the following commands: $ pipenv install gunicorn falcon psycopg2-binary git+https://github.com/alanorth/SolrClient.git@kazoo-2.5.0#egg=SolrClient $ pipenv install --dev ipython flake8 pytest	2018-11-14 09:07:32 +02:00
Alan Orth	a88600c92b	README.md: Add note about GPLv3	2018-11-13 12:34:31 +02:00
Alan Orth	019d9242c9	Merge pull request #7 from ilri/use-pip Rework to use pip instead of pipenv	2018-11-12 09:17:16 +02:00
Alan Orth	f4d7312a3f	CHANGELOG.md: Add unreleased changes	2018-11-12 09:02:04 +02:00
Alan Orth	9c46cfc7e2	Use Python 3.7 for pipenv Now that I'm only using pipenv locally it shouldn't create problems for people. They can still just create a vanilla virtualenv and use pip to install the dependencies.	2018-11-12 08:54:54 +02:00
Alan Orth	c1c2e319ac	README.md: Rework to use pip instead of pipenv Pipenv is great for local development, but I don't think many people are using it yet. I can use it locally and on Travis, but still keep vanilla requirements.txt for use with pip. The requirements.txt file can be generated easily from pipenv itself: $ pipenv lock -r > requirements.txt The same for the development requirements: $ pipenv lock -r -d > requirements-dev.txt	2018-11-12 08:49:02 +02:00
Alan Orth	0895b4f469	Add requirements-dev.txt for pip Generated with pipenv lock -r -d. Will be used for separating the development dependencies.	2018-11-12 08:48:45 +02:00
Alan Orth	dcfef06a65	Pipfile.lock: Run pipenv update	2018-11-12 08:20:47 +02:00