alanorth/dspace-statistics-api
1
0
Fork 0
mirror of https://github.com/ilri/dspace-statistics-api.git synced 2024-11-22 14:25:01 +01:00
Code Releases Activity

indexer.py: Major refactor

Browse Source 
Basically Solr's numFound has nothing to do with the actual number
of distinct facets that are returned. You need to use Solr's stats
component to get the number of distinct facets, aka countDistinct.
This is apparently deprecated in newer Solr versions, but we're on
version 4.10 and it works there.

Also, I realized that there is no need to return facets for items
without any views or downloads. Using facet.mincount=1 reduces the
result set size and also means we can store less data in the data-
base. The API returns HTTP 404 Not Found if an item is not in the
database anyways.

I can't figure it out exactly, but there is some weird issue with
Solr's facet results when you don't use facet.mincount=1. For some
reason you get tons of results with an id that doesn't even exist
in the document database, let alone as an actual DSpace item!

See: https://lucene.apache.org/solr/guide/6_6/the-stats-component.html
This commit is contained in:
Alan Orth 2018-09-26 02:41:10 +03:00
parent fc35b816f3
commit e604d8ca81
Signed by: alanorth
GPG Key ID: 0FB860CC9C45B1B9
1 changed files with 53 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
indexer.py
View File

@ -31,48 +31,62 @@
# See: https://wiki.duraspace.org/display/DSPACE/Solr # See: https://wiki.duraspace.org/display/DSPACE/Solr
from database import database_connection from database import database_connection
import json
from solr import solr_connection from solr import solr_connection
def index_views(): def index_views():
print("Populating database with item views.") # get total number of distinct facets for items with a minimum of 1 view,
# otherwise Solr returns all kinds of weird ids that are actually not in
# determine the total number of items with views (aka Solr's numFound) # the database. Also, stats are expensive, but we need stats.calcdistinct
# so we can get the countDistinct summary.
#
# see: https://lucene.apache.org/solr/guide/6_6/the-stats-component.html
res = solr.query('statistics', { res = solr.query('statistics', {
'q':'type:2', 'q':'type:2',
'fq':'isBot:false AND statistics_type:view', 'fq':'isBot:false AND statistics_type:view',
'facet':True, 'facet':True,
'facet.field':'id', 'facet.field':'id',
'facet.mincount':1,
'facet.limit':1,
'facet.offset':0,
'stats':True,
'stats.field':'id',
'stats.calcdistinct':True
}, rows=0) }, rows=0)
# divide results into "pages" (numFound / 100) # get total number of distinct facets (countDistinct)
results_numFound = res.get_num_found() results_totalNumFacets = json.loads(res.get_json())['stats']['stats_fields']['id']['countDistinct']
# divide results into "pages" (cast to int to effectively round down)
results_per_page = 100 results_per_page = 100
results_num_pages = round(results_numFound / results_per_page) results_num_pages = int(results_totalNumFacets / results_per_page)
results_current_page = 0 results_current_page = 0
cursor = db.cursor() cursor = db.cursor()
while results_current_page <= results_num_pages: while results_current_page <= results_num_pages:
print('Page {} of {}.'.format(results_current_page, results_num_pages)) print('Indexing item views (page {} of {})'.format(results_current_page, results_num_pages))
res = solr.query('statistics', { res = solr.query('statistics', {
'q':'type:2', 'q':'type:2',
'fq':'isBot:false AND statistics_type:view', 'fq':'isBot:false AND statistics_type:view',
'facet':True, 'facet':True,
'facet.field':'id', 'facet.field':'id',
'facet.mincount':1,
'facet.limit':results_per_page, 'facet.limit':results_per_page,
'facet.offset':results_current_page * results_per_page 'facet.offset':results_current_page * results_per_page
}) }, rows=0)
# make sure total number of results > 0 # check number of facets returned in the last query
if res.get_num_found() > 0: #results_currentNumFacets = len(res.get_facets()['id'])
# SolrClient's get_facets() returns a dict of dicts
views = res.get_facets() # SolrClient's get_facets() returns a dict of dicts
# in this case iterate over the 'id' dict and get the item ids and views views = res.get_facets()
for item_id, item_views in views['id'].items(): # in this case iterate over the 'id' dict and get the item ids and views
cursor.execute('''INSERT INTO items(id, views) VALUES(%s, %s) for item_id, item_views in views['id'].items():
ON CONFLICT(id) DO UPDATE SET downloads=excluded.views''', cursor.execute('''INSERT INTO items(id, views) VALUES(%s, %s)
(item_id, item_views)) ON CONFLICT(id) DO UPDATE SET downloads=excluded.views''',
(item_id, item_views))
db.commit() db.commit()
@ -81,45 +95,50 @@ def index_views():
cursor.close() cursor.close()
def index_downloads(): def index_downloads():
print("Populating database with item downloads.") # get the total number of distinct facets for items with at least 1 download
# determine the total number of items with downloads (aka Solr's numFound)
res = solr.query('statistics', { res = solr.query('statistics', {
'q':'type:0', 'q':'type:0',
'fq':'isBot:false AND statistics_type:view AND bundleName:ORIGINAL', 'fq':'isBot:false AND statistics_type:view AND bundleName:ORIGINAL',
'facet':True, 'facet':True,
'facet.field':'owningItem', 'facet.field':'owningItem',
'facet.mincount':1,
'facet.limit':1,
'facet.offset':0,
'stats':True,
'stats.field':'owningItem',
'stats.calcdistinct':True
}, rows=0) }, rows=0)
# divide results into "pages" (numFound / 100) # get total number of distinct facets (countDistinct)
results_numFound = res.get_num_found() results_totalNumFacets = json.loads(res.get_json())['stats']['stats_fields']['owningItem']['countDistinct']
# divide results into "pages" (cast to int to effectively round down)
results_per_page = 100 results_per_page = 100
results_num_pages = round(results_numFound / results_per_page) results_num_pages = int(results_totalNumFacets / results_per_page)
results_current_page = 0 results_current_page = 0
cursor = db.cursor() cursor = db.cursor()
while results_current_page <= results_num_pages: while results_current_page <= results_num_pages:
print('Page {} of {}.'.format(results_current_page, results_num_pages)) print('Indexing item downloads (page {} of {})'.format(results_current_page, results_num_pages))
res = solr.query('statistics', { res = solr.query('statistics', {
'q':'type:0', 'q':'type:0',
'fq':'isBot:false AND statistics_type:view AND bundleName:ORIGINAL', 'fq':'isBot:false AND statistics_type:view AND bundleName:ORIGINAL',
'facet':True, 'facet':True,
'facet.field':'owningItem', 'facet.field':'owningItem',
'facet.mincount':1,
'facet.limit':results_per_page, 'facet.limit':results_per_page,
'facet.offset':results_current_page * results_per_page 'facet.offset':results_current_page * results_per_page
}) }, rows=0)
# make sure total number of results > 0 # SolrClient's get_facets() returns a dict of dicts
if res.get_num_found() > 0: downloads = res.get_facets()
# SolrClient's get_facets() returns a dict of dicts # in this case iterate over the 'owningItem' dict and get the item ids and downloads
downloads = res.get_facets() for item_id, item_downloads in downloads['owningItem'].items():
# in this case iterate over the 'owningItem' dict and get the item ids and downloads cursor.execute('''INSERT INTO items(id, downloads) VALUES(%s, %s)
for item_id, item_downloads in downloads['owningItem'].items(): ON CONFLICT(id) DO UPDATE SET downloads=excluded.downloads''',
cursor.execute('''INSERT INTO items(id, downloads) VALUES(%s, %s) (item_id, item_downloads))
ON CONFLICT(id) DO UPDATE SET downloads=excluded.downloads''',
(item_id, item_downloads))
db.commit() db.commit()