CHANGELOG.md: Move unreleased notes to version 0.1.0

This route exposes all item statistics and uses the limit and offset
parameters to control paging throug the result set. The logic here
is extremely easy thanks to the brilliant LIMIT and OFFSET features
of SQLite (of course the SQL query sorts the results by some unique
field to ensure the order is already the same).

.gitignore

@@ -1,2 +1,3 @@
 __pycache__
 venv
+*.db

CHANGELOG.md

@@ -4,6 +4,37 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.1.0] - 2018-09-24
+### Changed
+- Rename project to "DSpace Statistics API"
+- Use read-only database connection in API
+- Update systemd units for CGSpace→DSpace rename
+- Use UPSERT to simplify database schema and Python logic
+
+### Added
+- Example systemd service and timer unit for indexer service
+- Add top-level route to expose all item statistics
+
+### Removed
+- Ability to customize SOLR_CORE variable
+
+## [0.0.4] - 2018-09-23
+### Added
+- Added example systemd unit file for API
+- Added indexer.py to ingest views and downloads from Solr to a SQLite database
+
+### Changed
+- Refactor Solr configuration and connection
+- /item route now expects id as part of the URI instead of a query parameter: /item/id
+- View and download stats are now fetched from a SQLite database
+
+## [0.0.3] - 2018-09-20
+### Changed
+- Refactor environment variables into config module
+- Simplify Solr query for "downloads"
+- Optimize Solr query by using rows=0
+- Fix Solr queries for item views
+
 ## [0.0.2] - 2018-09-18
 ### Added
 - Ability to get Solr parameters from environment (`SOLR_SERVER` and `SOLR_CORE`)

README.md

@@ -1,4 +1,4 @@
-# CGSpace Statistics API
+# DSpace Statistics API
 A quick and dirty REST API to expose Solr view and download statistics for items in a DSpace repository.
 
 Written and tested in Python 3.6. SolrClient (0.2.1) does not currently run in Python 3.7.0.
@@ -13,7 +13,10 @@ Create a virtual environment and run it:
 
 ## Todo
 
-- Take a list of items (POST in JSON?)
+- Add API documentation
+- Close up DB connection when gunicorn shuts down gracefully
+- Better logging
+- Return HTTP 404 when item_id is nonexistent
 
 ## License
 This work is licensed under the [GPLv3](https://www.gnu.org/licenses/gpl-3.0.en.html).

app.py

@@ -2,45 +2,67 @@
 # See DSpace Solr docs for tips about parameters
 # https://wiki.duraspace.org/display/DSPACE/Solr
 
+from database import database_connection_ro
 import falcon
-import os
-from SolrClient import SolrClient
+from solr import solr_connection
 
-# Check if Solr connection information was provided in the environment
-solr_server = os.environ.get('SOLR_SERVER', 'http://localhost:8080/solr')
-solr_core = os.environ.get('SOLR_CORE', 'statistics')
+db = database_connection_ro()
+solr = solr_connection()
 
-class ItemResource:
+class AllItemsResource:
     def on_get(self, req, resp):
         """Handles GET requests"""
         # Return HTTPBadRequest if id parameter is not present and valid
-        item_id = req.get_param_as_int("id", required=True, min=0)
+        limit = req.get_param_as_int("limit", min=0, max=100) or 100
+        page = req.get_param_as_int("page", min=0) or 0
+        offset = limit * page
 
-        solr = SolrClient(solr_server)
+        cursor = db.cursor()
 
-        # Get views
-        res = solr.query(solr_core, {
-            'q':'type:0',
-            'fq':'owningItem:{0} AND isBot:false AND statistics_type:view AND -bundleName:ORIGINAL'.format(item_id)
-        })
+        # get total number of items so we can estimate the pages
+        cursor.execute('SELECT COUNT(id) FROM items')
+        pages = round(cursor.fetchone()[0] / limit)
 
-        views = res.get_num_found()
+        # get statistics, ordered by id, and use limit and offset to page through results
+        cursor.execute('SELECT id, views, downloads FROM items ORDER BY id ASC LIMIT {0} OFFSET {1}'.format(limit, offset))
+        results = cursor.fetchmany(limit)
+        cursor.close()
 
-        # Get downloads
-        res = solr.query(solr_core, {
-            'q':'type:0',
-            'fq':'owningItem:{0} AND isBot:false AND statistics_type:view AND -(bundleName:[* TO *] -bundleName:ORIGINAL)'.format(item_id)
-        })
+        # create a list to hold dicts of item stats
+        statistics = list()
 
-        downloads = res.get_num_found() 
+        # iterate over results and build statistics object
+        for item in results:
+            statistics.append({ 'id': item['id'], 'views': item['views'], 'downloads': item['downloads'] })
+
+        message = {
+                'currentPage': page,
+                'totalPages': pages,
+                'limit': limit,
+                'statistics': statistics
+        }
+
+        resp.media = message
+
+class ItemResource:
+    def on_get(self, req, resp, item_id):
+        """Handles GET requests"""
+
+        cursor = db.cursor()
+        cursor.execute('SELECT views, downloads FROM items WHERE id={0}'.format(item_id))
+        results = cursor.fetchone()
+        cursor.close()
 
         statistics = {
             'id': item_id,
-            'views': views,
-            'downloads': downloads
+            'views': results['views'],
+            'downloads': results['downloads']
         }
 
         resp.media = statistics
 
 api = falcon.API()
-api.add_route('/item', ItemResource())
+api.add_route('/', AllItemsResource())
+api.add_route('/item/{item_id:int}', ItemResource())
+
+# vim: set sw=4 ts=4 expandtab:

config.py

@@ -0,0 +1,8 @@
+import os
+
+# Check if Solr connection information was provided in the environment
+SOLR_SERVER = os.environ.get('SOLR_SERVER', 'http://localhost:8080/solr')
+
+SQLITE_DB = os.environ.get('SQLITE_DB', 'statistics.db')
+
+# vim: set sw=4 ts=4 expandtab:

contrib/dspace-statistics-api.service

@@ -0,0 +1,17 @@
+[Unit]
+Description=DSpace Statistics API
+After=network.target
+
+[Service]
+Environment=SOLR_SERVER=http://localhost:8081/solr
+User=nobody
+Group=nogroup
+WorkingDirectory=/opt/ilri/dspace-statistics-api
+ExecStart=/opt/ilri/dspace-statistics-api/venv/bin/gunicorn \
+          --bind 127.0.0.1:5000                             \
+          app:api
+ExecReload=/bin/kill -s HUP $MAINPID
+ExecStop=/bin/kill -s TERM $MAINPID
+
+[Install]
+WantedBy=multi-user.target

contrib/dspace-statistics-indexer.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=DSpace Statistics Indexer
+After=tomcat7.target
+
+[Service]
+Environment=SOLR_SERVER=http://localhost:8081/solr
+User=nobody
+Group=nogroup
+WorkingDirectory=/opt/ilri/dspace-statistics-api
+ExecStart=/opt/ilri/dspace-statistics-api/venv/bin/python indexer.py
+
+[Install]
+WantedBy=multi-user.target

contrib/dspace-statistics-indexer.timer

@@ -0,0 +1,12 @@
+[Unit]
+Description=DSpace Statistics Indexer
+
+[Timer]
+# twice a day, at 6AM and 6PM
+OnCalendar=*-*-* 06:00:00,18:00:00
+# Add a random delay of 0–3600 seconds
+RandomizedDelaySec=3600
+Persistent=true
+
+[Install]
+WantedBy=timers.target

database.py

@@ -0,0 +1,18 @@
+from config import SQLITE_DB
+import sqlite3
+
+def database_connection_rw():
+    connection = sqlite3.connect(SQLITE_DB)
+    # allow iterating over row results by column key
+    connection.row_factory = sqlite3.Row
+
+    return connection
+
+def database_connection_ro():
+    connection = sqlite3.connect('file:{0}?mode=ro'.format(SQLITE_DB), uri=True)
+    # allow iterating over row results by column key
+    connection.row_factory = sqlite3.Row
+
+    return connection
+
+# vim: set sw=4 ts=4 expandtab:

indexer.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python
+#
+# indexer.py
+#
+# Copyright 2018 Alan Orth.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# ---
+#
+# Connects to a DSpace Solr statistics core and ingests item views and downloads
+# into a SQLite database for use with other applications (an API, for example).
+#
+# This script is written for Python 3 and requires several modules that you can
+# install with pip (I recommend setting up a Python virtual environment first):
+#
+#   $ pip install SolrClient
+#
+# See: https://solrclient.readthedocs.io/en/latest/SolrClient.html
+# See: https://wiki.duraspace.org/display/DSPACE/Solr
+#
+# Tested with Python 3.5 and 3.6.
+
+from database import database_connection_rw
+from solr import solr_connection
+
+def index_views():
+    print("Populating database with item views.")
+
+    # determine the total number of items with views (aka Solr's numFound)
+    res = solr.query('statistics', {
+        'q':'type:2',
+        'fq':'isBot:false AND statistics_type:view',
+        'facet':True,
+        'facet.field':'id',
+    }, rows=0)
+
+    # divide results into "pages" (numFound / 100)
+    results_numFound = res.get_num_found()
+    results_per_page = 100
+    results_num_pages = round(results_numFound / results_per_page)
+    results_current_page = 0
+
+    while results_current_page <= results_num_pages:
+        print('Page {0} of {1}.'.format(results_current_page, results_num_pages))
+
+        res = solr.query('statistics', {
+            'q':'type:2',
+            'fq':'isBot:false AND statistics_type:view',
+            'facet':True,
+            'facet.field':'id',
+            'facet.limit':results_per_page,
+            'facet.offset':results_current_page * results_per_page
+        })
+
+        # make sure total number of results > 0
+        if res.get_num_found() > 0:
+            # SolrClient's get_facets() returns a dict of dicts
+            views = res.get_facets()
+            # in this case iterate over the 'id' dict and get the item ids and views
+            for item_id, item_views in views['id'].items():
+                db.execute('''INSERT INTO items(id, views) VALUES(?, ?)
+                               ON CONFLICT(id) DO UPDATE SET downloads=excluded.views''',
+                               (item_id, item_views))
+
+        db.commit()
+
+        results_current_page += 1
+
+def index_downloads():
+    print("Populating database with item downloads.")
+
+    # determine the total number of items with downloads (aka Solr's numFound)
+    res = solr.query('statistics', {
+        'q':'type:0',
+        'fq':'isBot:false AND statistics_type:view AND bundleName:ORIGINAL',
+        'facet':True,
+        'facet.field':'owningItem',
+    }, rows=0)
+
+    # divide results into "pages" (numFound / 100)
+    results_numFound = res.get_num_found()
+    results_per_page = 100
+    results_num_pages = round(results_numFound / results_per_page)
+    results_current_page = 0
+
+    while results_current_page <= results_num_pages:
+        print('Page {0} of {1}.'.format(results_current_page, results_num_pages))
+
+        res = solr.query('statistics', {
+            'q':'type:0',
+            'fq':'isBot:false AND statistics_type:view AND bundleName:ORIGINAL',
+            'facet':True,
+            'facet.field':'owningItem',
+            'facet.limit':results_per_page,
+            'facet.offset':results_current_page * results_per_page
+        })
+
+        # make sure total number of results > 0
+        if res.get_num_found() > 0:
+            # SolrClient's get_facets() returns a dict of dicts
+            downloads = res.get_facets()
+            # in this case iterate over the 'owningItem' dict and get the item ids and downloads
+            for item_id, item_downloads in downloads['owningItem'].items():
+                db.execute('''INSERT INTO items(id, downloads) VALUES(?, ?)
+                               ON CONFLICT(id) DO UPDATE SET downloads=excluded.downloads''',
+                               (item_id, item_downloads))
+
+        db.commit()
+
+        results_current_page += 1
+
+db = database_connection_rw()
+solr = solr_connection()
+
+# create table to store item views and downloads
+db.execute('''CREATE TABLE IF NOT EXISTS items
+                  (id INT PRIMARY KEY, views INT DEFAULT 0, downloads INT DEFAULT 0)''')
+index_views()
+index_downloads()
+
+db.close()
+
+# vim: set sw=4 ts=4 expandtab:

solr.py

@@ -0,0 +1,9 @@
+from config import SOLR_SERVER
+from SolrClient import SolrClient
+
+def solr_connection():
+    connection = SolrClient(SOLR_SERVER)
+
+    return connection
+
+# vim: set sw=4 ts=4 expandtab: