CHANGELOG.md: Move entries to version 0.0.4

Standalone script to ingest item views and downloads from Solr into
SQLite.

.gitignore

@@ -1,2 +1,3 @@
 __pycache__
 venv
+*.db

CHANGELOG.md

@@ -4,6 +4,23 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.0.4] - 2018-09-23
+### Added
+- Added example systemd unit file for API
+- Added indexer.py to ingest views and downloads from Solr to a SQLite database
+
+### Changed
+- Refactor Solr configuration and connection
+- /item route now expects id as part of the URI instead of a query parameter: /item/id
+- View and download stats are now fetched from a SQLite database
+
+## [0.0.3] - 2018-09-20
+### Changed
+- Refactor environment variables into config module
+- Simplify Solr query for "downloads"
+- Optimize Solr query by using rows=0
+- Fix Solr queries for item views
+
 ## [0.0.2] - 2018-09-18
 ### Added
 - Ability to get Solr parameters from environment (`SOLR_SERVER` and `SOLR_CORE`)

README.md

@@ -13,7 +13,8 @@ Create a virtual environment and run it:
 
 ## Todo
 
-- Take a list of items (POST in JSON?)
+- Ability to return a paginated list of items (on a different route?)
+- Add API documentation
 
 ## License
 This work is licensed under the [GPLv3](https://www.gnu.org/licenses/gpl-3.0.en.html).

app.py

@@ -2,37 +2,34 @@
 # See DSpace Solr docs for tips about parameters
 # https://wiki.duraspace.org/display/DSPACE/Solr
 
+from config import SOLR_CORE
+from database import database_connection
 import falcon
-import os
+from solr import solr_connection
-from SolrClient import SolrClient
 
-# Check if Solr connection information was provided in the environment
+db = database_connection()
-solr_server = os.environ.get('SOLR_SERVER', 'http://localhost:8080/solr')
+solr = solr_connection()
-solr_core = os.environ.get('SOLR_CORE', 'statistics')
 
 class ItemResource:
-    def on_get(self, req, resp):
+    def on_get(self, req, resp, item_id):
         """Handles GET requests"""
-        # Return HTTPBadRequest if id parameter is not present and valid
-        item_id = req.get_param_as_int("id", required=True, min=0)
 
-        solr = SolrClient(solr_server)
+        cursor = db.cursor()
+        # get item views (and catch the TypeError if item doesn't have any views)
+        cursor.execute('SELECT views FROM itemviews WHERE id={0}'.format(item_id))
+        try:
+            views = cursor.fetchone()['views']
+        except:
+            views = 0
 
-        # Get views
+        # get item downloads (and catch the TypeError if item doesn't have any downloads)
-        res = solr.query(solr_core, {
+        cursor.execute('SELECT downloads FROM itemdownloads WHERE id={0}'.format(item_id))
-            'q':'type:0',
+        try:
-            'fq':'owningItem:{0} AND isBot:false AND statistics_type:view AND -bundleName:ORIGINAL'.format(item_id)
+            downloads = cursor.fetchone()['downloads']
-        })
+        except:
+            downloads = 0
 
-        views = res.get_num_found()
+        cursor.close()
-
-        # Get downloads
-        res = solr.query(solr_core, {
-            'q':'type:0',
-            'fq':'owningItem:{0} AND isBot:false AND statistics_type:view AND -(bundleName:[* TO *] -bundleName:ORIGINAL)'.format(item_id)
-        })
-
-        downloads = res.get_num_found() 
 
         statistics = {
             'id': item_id,
@@ -43,4 +40,6 @@ class ItemResource:
         resp.media = statistics
 
 api = falcon.API()
-api.add_route('/item', ItemResource())
+api.add_route('/item/{item_id:int}', ItemResource())
+
+# vim: set sw=4 ts=4 expandtab:

config.py

@@ -0,0 +1,9 @@
+import os
+
+# Check if Solr connection information was provided in the environment
+SOLR_SERVER = os.environ.get('SOLR_SERVER', 'http://localhost:8080/solr')
+SOLR_CORE = os.environ.get('SOLR_CORE', 'statistics')
+
+SQLITE_DB = os.environ.get('SQLITE_DB', 'statistics.db')
+
+# vim: set sw=4 ts=4 expandtab:

contrib/cgspace-statistics-api.service

@@ -0,0 +1,18 @@
+[Unit]
+Description=CGSpace Statistics API
+After=network.target
+
+[Service]
+Environment=SOLR_SERVER=http://localhost:8081/solr
+Environment=SOLR_CORE=statistics
+User=nobody
+Group=nogroup
+WorkingDirectory=/opt/ilri/cgspace-statistics-api
+ExecStart=/opt/ilri/cgspace-statistics-api/venv/bin/gunicorn \
+          --bind 127.0.0.1:5000                              \
+          app:api
+ExecReload=/bin/kill -s HUP $MAINPID
+ExecStop=/bin/kill -s TERM $MAINPID
+
+[Install]
+WantedBy=multi-user.target

database.py

@@ -0,0 +1,11 @@
+from config import SQLITE_DB
+import sqlite3
+
+def database_connection():
+    connection = sqlite3.connect(SQLITE_DB)
+    # allow iterating over row results by column key
+    connection.row_factory = sqlite3.Row
+
+    return connection
+
+# vim: set sw=4 ts=4 expandtab:

indexer.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python
+#
+# Tested with Python 3.6
+# See DSpace Solr docs for tips about parameters
+# https://wiki.duraspace.org/display/DSPACE/Solr
+
+from config import SOLR_CORE
+from database import database_connection
+from solr import solr_connection
+
+def index_views():
+    print("Populating database with item views.")
+
+    # determine the total number of items with views (aka Solr's numFound)
+    res = solr.query(SOLR_CORE, {
+        'q':'type:2',
+        'fq':'isBot:false AND statistics_type:view',
+        'facet':True,
+        'facet.field':'id',
+    }, rows=0)
+
+    # divide results into "pages" (numFound / 100)
+    results_numFound = res.get_num_found()
+    results_per_page = 100
+    results_num_pages = round(results_numFound / results_per_page)
+    results_current_page = 0
+
+    while results_current_page <= results_num_pages:
+        print('Page {0} of {1}.'.format(results_current_page, results_num_pages))
+
+        res = solr.query(SOLR_CORE, {
+            'q':'type:2',
+            'fq':'isBot:false AND statistics_type:view',
+            'facet':True,
+            'facet.field':'id',
+            'facet.limit':results_per_page,
+            'facet.offset':results_current_page * results_per_page
+        })
+
+        # make sure total number of results > 0
+        if res.get_num_found() > 0:
+            # SolrClient's get_facets() returns a dict of dicts
+            views = res.get_facets()
+            # in this case iterate over the 'id' dict and get the item ids and views
+            for item_id, item_views in views['id'].items():
+                db.execute('''REPLACE INTO itemviews VALUES (?, ?)''', (item_id, item_views))
+
+        db.commit()
+
+        results_current_page += 1
+
+def index_downloads():
+    print("Populating database with item downloads.")
+
+    # determine the total number of items with downloads (aka Solr's numFound)
+    res = solr.query(SOLR_CORE, {
+        'q':'type:0',
+        'fq':'isBot:false AND statistics_type:view AND bundleName:ORIGINAL',
+        'facet':True,
+        'facet.field':'owningItem',
+    }, rows=0)
+
+    # divide results into "pages" (numFound / 100)
+    results_numFound = res.get_num_found()
+    results_per_page = 100
+    results_num_pages = round(results_numFound / results_per_page)
+    results_current_page = 0
+
+    while results_current_page <= results_num_pages:
+        print('Page {0} of {1}.'.format(results_current_page, results_num_pages))
+
+        res = solr.query(SOLR_CORE, {
+            'q':'type:0',
+            'fq':'isBot:false AND statistics_type:view AND bundleName:ORIGINAL',
+            'facet':True,
+            'facet.field':'owningItem',
+            'facet.limit':results_per_page,
+            'facet.offset':results_current_page * results_per_page
+        })
+
+        # make sure total number of results > 0
+        if res.get_num_found() > 0:
+            # SolrClient's get_facets() returns a dict of dicts
+            downloads = res.get_facets()
+            # in this case iterate over the 'owningItem' dict and get the item ids and downloads
+            for item_id, item_downloads in downloads['owningItem'].items():
+                db.execute('''REPLACE INTO itemdownloads VALUES (?, ?)''', (item_id, item_downloads))
+
+        db.commit()
+
+        results_current_page += 1
+
+db = database_connection()
+solr = solr_connection()
+
+# use separate views and downloads tables so we can REPLACE INTO carelessly (ie, item may have views but no downloads)
+db.execute('''CREATE TABLE IF NOT EXISTS itemviews
+                  (id integer primary key, views integer)''')
+db.execute('''CREATE TABLE IF NOT EXISTS itemdownloads
+                  (id integer primary key, downloads integer)''')
+index_views()
+index_downloads()
+
+db.close()
+
+# vim: set sw=4 ts=4 expandtab:

solr.py

@@ -0,0 +1,9 @@
+from config import SOLR_SERVER
+from SolrClient import SolrClient
+
+def solr_connection():
+    connection = SolrClient(SOLR_SERVER)
+
+    return connection
+
+# vim: set sw=4 ts=4 expandtab: