CHANGELOG.md: Release version 0.2.1

This comment was added when I first began the application and the
testing status is documented in the README now.

CHANGELOG.md

@@ -4,6 +4,39 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.2.1] - 2018-09-24
+### Changed
+- Environment settings in example systemd unit files
+- Use psycopg2.extras.DictCursor for PostgreSQL connection
+
+## [0.2.0] - 2018-09-24
+### Changed
+- Use PostgreSQL instead of SQLite because UPSERT support needs a very new libsqlite3 whereas it's already in PostgreSQL 9.5+
+
+## [0.1.0] - 2018-09-24
+### Changed
+- Rename project to "DSpace Statistics API"
+- Use read-only database connection in API
+- Update systemd units for CGSpace→DSpace rename
+- Use UPSERT to simplify database schema and Python logic
+
+### Added
+- Example systemd service and timer unit for indexer service
+- Add top-level route to expose all item statistics
+
+### Removed
+- Ability to customize SOLR_CORE variable
+
+## [0.0.4] - 2018-09-23
+### Added
+- Added example systemd unit file for API
+- Added indexer.py to ingest views and downloads from Solr to a SQLite database
+
+### Changed
+- Refactor Solr configuration and connection
+- /item route now expects id as part of the URI instead of a query parameter: /item/id
+- View and download stats are now fetched from a SQLite database
+
 ## [0.0.3] - 2018-09-20
 ### Changed
 - Refactor environment variables into config module

README.md

@@ -1,19 +1,22 @@
-# CGSpace Statistics API
+# DSpace Statistics API
 A quick and dirty REST API to expose Solr view and download statistics for items in a DSpace repository.
 
-Written and tested in Python 3.6. SolrClient (0.2.1) does not currently run in Python 3.7.0.
+Written and tested in Python 3.6. SolrClient (0.2.1) does not currently run in Python 3.7.0. Requires PostgreSQL version 9.5 or greater for [`UPSERT` support](https://wiki.postgresql.org/wiki/UPSERT).
 
 ## Installation
 Create a virtual environment and run it:
 
     $ virtualenv -p /usr/bin/python3.6 venv
     $ . venv/bin/activate
-    $ pip install falcon gunicorn SolrClient
+    $ pip install falcon gunicorn SolrClient psycopg2-binary
     $ gunicorn app:api
 
 ## Todo
 
-- Ability to return a paginated list of items (on a different route?)
+- Add API documentation
+- Close up DB connection when gunicorn shuts down gracefully
+- Better logging
+- Return HTTP 404 when item_id is nonexistent
 
 ## License
 This work is licensed under the [GPLv3](https://www.gnu.org/licenses/gpl-3.0.en.html).

app.py

@@ -1,44 +1,65 @@
-# Tested with Python 3.6
-# See DSpace Solr docs for tips about parameters
-# https://wiki.duraspace.org/display/DSPACE/Solr
-
-from config import SOLR_SERVER
-from config import SOLR_CORE
+from database import database_connection
 import falcon
-from SolrClient import SolrClient
+from solr import solr_connection
 
+db = database_connection()
+db.set_session(readonly=True)
+solr = solr_connection()
 
-class ItemResource:
+class AllItemsResource:
     def on_get(self, req, resp):
         """Handles GET requests"""
         # Return HTTPBadRequest if id parameter is not present and valid
-        item_id = req.get_param_as_int("id", required=True, min=0)
+        limit = req.get_param_as_int("limit", min=0, max=100) or 100
+        page = req.get_param_as_int("page", min=0) or 0
+        offset = limit * page
 
-        solr = SolrClient(SOLR_SERVER)
+        cursor = db.cursor()
 
-        # Get views
-        res = solr.query(SOLR_CORE, {
-            'q':'type:2 AND id:{0}'.format(item_id),
-            'fq':'isBot:false AND statistics_type:view'
-        }, rows=0)
+        # get total number of items so we can estimate the pages
+        cursor.execute('SELECT COUNT(id) FROM items')
+        pages = round(cursor.fetchone()[0] / limit)
 
-        views = res.get_num_found()
+        # get statistics, ordered by id, and use limit and offset to page through results
+        cursor.execute('SELECT id, views, downloads FROM items ORDER BY id ASC LIMIT {} OFFSET {}'.format(limit, offset))
+        results = cursor.fetchmany(limit)
+        cursor.close()
 
-        # Get downloads
-        res = solr.query(SOLR_CORE, {
-            'q':'type:0 AND owningItem:{0}'.format(item_id),
-            'fq':'isBot:false AND statistics_type:view AND bundleName:ORIGINAL'
-        }, rows=0)
+        # create a list to hold dicts of item stats
+        statistics = list()
 
-        downloads = res.get_num_found() 
+        # iterate over results and build statistics object
+        for item in results:
+            statistics.append({ 'id': item['id'], 'views': item['views'], 'downloads': item['downloads'] })
+
+        message = {
+                'currentPage': page,
+                'totalPages': pages,
+                'limit': limit,
+                'statistics': statistics
+        }
+
+        resp.media = message
+
+class ItemResource:
+    def on_get(self, req, resp, item_id):
+        """Handles GET requests"""
+
+        cursor = db.cursor()
+        cursor.execute('SELECT views, downloads FROM items WHERE id={}'.format(item_id))
+        results = cursor.fetchone()
+        cursor.close()
 
         statistics = {
             'id': item_id,
-            'views': views,
-            'downloads': downloads
+            'views': results['views'],
+            'downloads': results['downloads']
         }
 
         resp.media = statistics
 
 api = falcon.API()
-api.add_route('/item', ItemResource())
+api.add_route('/', AllItemsResource())
+api.add_route('/item/{item_id:int}', ItemResource())
+
+# vim: set sw=4 ts=4 expandtab:

config.py

@@ -2,4 +2,10 @@ import os
 
 # Check if Solr connection information was provided in the environment
 SOLR_SERVER = os.environ.get('SOLR_SERVER', 'http://localhost:8080/solr')
-SOLR_CORE = os.environ.get('SOLR_CORE', 'statistics')
+
+DATABASE_NAME = os.environ.get('DATABASE_NAME', 'dspacestatistics')
+DATABASE_USER = os.environ.get('DATABASE_USER', 'dspacestatistics')
+DATABASE_PASS = os.environ.get('DATABASE_PASS', 'dspacestatistics')
+DATABASE_HOST = os.environ.get('DATABASE_HOST', 'localhost')
+
+# vim: set sw=4 ts=4 expandtab:

contrib/dspace-statistics-api.service

@@ -0,0 +1,20 @@
+[Unit]
+Description=DSpace Statistics API
+After=network.target
+
+[Service]
+Environment=DATABASE_NAME=dspacestatistics
+Environment=DATABASE_USER=dspacestatistics
+Environment=DATABASE_PASS=dspacestatistics
+Environment=DATABASE_HOST=localhost
+User=nobody
+Group=nogroup
+WorkingDirectory=/opt/ilri/dspace-statistics-api
+ExecStart=/opt/ilri/dspace-statistics-api/venv/bin/gunicorn \
+          --bind 127.0.0.1:5000                             \
+          app:api
+ExecReload=/bin/kill -s HUP $MAINPID
+ExecStop=/bin/kill -s TERM $MAINPID
+
+[Install]
+WantedBy=multi-user.target

contrib/dspace-statistics-indexer.service

@@ -0,0 +1,17 @@
+[Unit]
+Description=DSpace Statistics Indexer
+After=tomcat7.target
+
+[Service]
+Environment=SOLR_SERVER=http://localhost:8081/solr
+Environment=DATABASE_NAME=dspacestatistics
+Environment=DATABASE_USER=dspacestatistics
+Environment=DATABASE_PASS=dspacestatistics
+Environment=DATABASE_HOST=localhost
+User=nobody
+Group=nogroup
+WorkingDirectory=/opt/ilri/dspace-statistics-api
+ExecStart=/opt/ilri/dspace-statistics-api/venv/bin/python indexer.py
+
+[Install]
+WantedBy=multi-user.target

contrib/dspace-statistics-indexer.timer

@@ -0,0 +1,12 @@
+[Unit]
+Description=DSpace Statistics Indexer
+
+[Timer]
+# twice a day, at 6AM and 6PM
+OnCalendar=*-*-* 06:00:00,18:00:00
+# Add a random delay of 0–3600 seconds
+RandomizedDelaySec=3600
+Persistent=true
+
+[Install]
+WantedBy=timers.target

database.py

@@ -0,0 +1,13 @@
+from config import DATABASE_NAME
+from config import DATABASE_USER
+from config import DATABASE_PASS
+from config import DATABASE_HOST
+import psycopg2
+import psycopg2.extras
+
+def database_connection():
+    connection = psycopg2.connect("dbname={} user={} password={} host='{}'".format(DATABASE_NAME, DATABASE_USER, DATABASE_PASS, DATABASE_HOST), cursor_factory=psycopg2.extras.DictCursor)
+
+    return connection
+
+# vim: set sw=4 ts=4 expandtab:

indexer.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python
+#
+# indexer.py
+#
+# Copyright 2018 Alan Orth.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# ---
+#
+# Connects to a DSpace Solr statistics core and ingests item views and downloads
+# into a Postgres database for use with other applications (an API, for example).
+#
+# This script is written for Python 3 and requires several modules that you can
+# install with pip (I recommend setting up a Python virtual environment first):
+#
+#   $ pip install SolrClient
+#
+# See: https://solrclient.readthedocs.io/en/latest/SolrClient.html
+# See: https://wiki.duraspace.org/display/DSPACE/Solr
+#
+# Tested with Python 3.5 and 3.6.
+
+from database import database_connection
+from solr import solr_connection
+
+def index_views():
+    print("Populating database with item views.")
+
+    # determine the total number of items with views (aka Solr's numFound)
+    res = solr.query('statistics', {
+        'q':'type:2',
+        'fq':'isBot:false AND statistics_type:view',
+        'facet':True,
+        'facet.field':'id',
+    }, rows=0)
+
+    # divide results into "pages" (numFound / 100)
+    results_numFound = res.get_num_found()
+    results_per_page = 100
+    results_num_pages = round(results_numFound / results_per_page)
+    results_current_page = 0
+
+    cursor = db.cursor()
+
+    while results_current_page <= results_num_pages:
+        print('Page {} of {}.'.format(results_current_page, results_num_pages))
+
+        res = solr.query('statistics', {
+            'q':'type:2',
+            'fq':'isBot:false AND statistics_type:view',
+            'facet':True,
+            'facet.field':'id',
+            'facet.limit':results_per_page,
+            'facet.offset':results_current_page * results_per_page
+        })
+
+        # make sure total number of results > 0
+        if res.get_num_found() > 0:
+            # SolrClient's get_facets() returns a dict of dicts
+            views = res.get_facets()
+            # in this case iterate over the 'id' dict and get the item ids and views
+            for item_id, item_views in views['id'].items():
+                cursor.execute('''INSERT INTO items(id, views) VALUES(%s, %s)
+                               ON CONFLICT(id) DO UPDATE SET downloads=excluded.views''',
+                               (item_id, item_views))
+
+        db.commit()
+
+        results_current_page += 1
+
+    cursor.close()
+
+def index_downloads():
+    print("Populating database with item downloads.")
+
+    # determine the total number of items with downloads (aka Solr's numFound)
+    res = solr.query('statistics', {
+        'q':'type:0',
+        'fq':'isBot:false AND statistics_type:view AND bundleName:ORIGINAL',
+        'facet':True,
+        'facet.field':'owningItem',
+    }, rows=0)
+
+    # divide results into "pages" (numFound / 100)
+    results_numFound = res.get_num_found()
+    results_per_page = 100
+    results_num_pages = round(results_numFound / results_per_page)
+    results_current_page = 0
+
+    cursor = db.cursor()
+
+    while results_current_page <= results_num_pages:
+        print('Page {} of {}.'.format(results_current_page, results_num_pages))
+
+        res = solr.query('statistics', {
+            'q':'type:0',
+            'fq':'isBot:false AND statistics_type:view AND bundleName:ORIGINAL',
+            'facet':True,
+            'facet.field':'owningItem',
+            'facet.limit':results_per_page,
+            'facet.offset':results_current_page * results_per_page
+        })
+
+        # make sure total number of results > 0
+        if res.get_num_found() > 0:
+            # SolrClient's get_facets() returns a dict of dicts
+            downloads = res.get_facets()
+            # in this case iterate over the 'owningItem' dict and get the item ids and downloads
+            for item_id, item_downloads in downloads['owningItem'].items():
+                cursor.execute('''INSERT INTO items(id, downloads) VALUES(%s, %s)
+                               ON CONFLICT(id) DO UPDATE SET downloads=excluded.downloads''',
+                               (item_id, item_downloads))
+
+        db.commit()
+
+        results_current_page += 1
+
+    cursor.close()
+
+db = database_connection()
+solr = solr_connection()
+
+# create table to store item views and downloads
+cursor = db.cursor()
+cursor.execute('''CREATE TABLE IF NOT EXISTS items
+                  (id INT PRIMARY KEY, views INT DEFAULT 0, downloads INT DEFAULT 0)''')
+index_views()
+index_downloads()
+
+db.close()
+
+# vim: set sw=4 ts=4 expandtab:

solr.py

@@ -0,0 +1,9 @@
+from config import SOLR_SERVER
+from SolrClient import SolrClient
+
+def solr_connection():
+    connection = SolrClient(SOLR_SERVER)
+
+    return connection
+
+# vim: set sw=4 ts=4 expandtab: