mirror of
https://github.com/ilri/dspace-statistics-api.git
synced 2025-05-10 15:16:02 +02:00
Compare commits
28 Commits
Author | SHA1 | Date | |
---|---|---|---|
963961354b
|
|||
1b394ec50e
|
|||
3e9753b600
|
|||
cb3c3d37fa
|
|||
4ff1fd4a22
|
|||
d2fe420a9a
|
|||
3197b79578
|
|||
eeb8e6bba1
|
|||
3540ce328b
|
|||
520e04f9be
|
|||
8a46a64cfc
|
|||
b8442f8cce
|
|||
95f7871cc1
|
|||
3bc07027e5
|
|||
afcc445855
|
|||
494548c691
|
|||
feb60b6adf
|
|||
1541ae3e3b
|
|||
1aedc0ca29
|
|||
a648183f35
|
|||
b8f379e7fa
|
|||
78f9949ecb
|
|||
af80c4b447
|
|||
edd9e90f59
|
|||
1806d50a51
|
|||
a459e66fd9
|
|||
5a3b392a1d | |||
9dcda114c6 |
10
.travis.yml
10
.travis.yml
@ -1,9 +1,10 @@
|
||||
dist: xenial
|
||||
dist: bionic
|
||||
language: python
|
||||
python:
|
||||
- "3.5"
|
||||
- "3.6"
|
||||
- "3.7"
|
||||
- "3.8"
|
||||
addons:
|
||||
postgresql: "9.6"
|
||||
before_script:
|
||||
@ -13,11 +14,8 @@ before_script:
|
||||
- createdb -U postgres -O dspacestatistics --encoding=UNICODE dspacestatistics
|
||||
- psql -U postgres -d dspacestatistics < tests/dspacestatistics.sql
|
||||
install:
|
||||
- "pip install pipenv --upgrade-strategy=only-if-needed"
|
||||
- "pipenv install --dev"
|
||||
- "pip install -r requirements.txt"
|
||||
- "pip install -r requirements-dev.txt"
|
||||
script: pytest
|
||||
env:
|
||||
- PIPENV_NOSPIN=True
|
||||
- PIPENV_HIDE_EMOJIS=True
|
||||
|
||||
# vim: ts=2 sw=2 et
|
||||
|
18
CHANGELOG.md
18
CHANGELOG.md
@ -4,6 +4,24 @@ All notable changes to this project will be documented in this file.
|
||||
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
||||
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
||||
|
||||
## [1.1.1] - 2019-11-27
|
||||
### Added
|
||||
- Configuration for automatic sorting of imports with isort
|
||||
- Configuration for automatic code formatting with black
|
||||
|
||||
### Updated
|
||||
- Run pipenv update, bringing psycogpg 2.8.3, requests 2.22.0, pytest 5.3.1,
|
||||
and gunicorn 20.0.4
|
||||
|
||||
### Changed
|
||||
- Use Ubuntu 18.04 "Bionic" for TravisCI builds
|
||||
- Use Python 3.8.0 for pipenv
|
||||
- Minor syntax issues highlighted by flake8
|
||||
|
||||
## [1.1.0] - 2019-05-05
|
||||
## Updated
|
||||
- Falcon 2.0.0 (@alanorth)
|
||||
|
||||
## [1.0.0] - 2019-04-15
|
||||
### Added
|
||||
- Build configuration for build.sr.ht
|
||||
|
6
Pipfile
6
Pipfile
@ -5,7 +5,7 @@ name = "pypi"
|
||||
|
||||
[packages]
|
||||
gunicorn = "*"
|
||||
falcon = "*"
|
||||
falcon = "==2.0.0"
|
||||
"psycopg2-binary" = "*"
|
||||
requests = "*"
|
||||
|
||||
@ -13,6 +13,8 @@ requests = "*"
|
||||
ipython = "*"
|
||||
"flake8" = "*"
|
||||
pytest = "*"
|
||||
isort = "*"
|
||||
black = "*"
|
||||
|
||||
[requires]
|
||||
python_version = "3.7"
|
||||
python_version = "3.8"
|
||||
|
304
Pipfile.lock
generated
304
Pipfile.lock
generated
@ -1,11 +1,11 @@
|
||||
{
|
||||
"_meta": {
|
||||
"hash": {
|
||||
"sha256": "d18369b55f85594f6acfa49779909334a794a4002656508fc85acea43df2521d"
|
||||
"sha256": "5e3f3e80bd92780ab66a80a83156001e50362a0d6195b1a304a15dc0c645c562"
|
||||
},
|
||||
"pipfile-spec": 6,
|
||||
"requires": {
|
||||
"python_version": "3.7"
|
||||
"python_version": "3.8"
|
||||
},
|
||||
"sources": [
|
||||
{
|
||||
@ -18,10 +18,10 @@
|
||||
"default": {
|
||||
"certifi": {
|
||||
"hashes": [
|
||||
"sha256:59b7658e26ca9c7339e00f8f4636cdfe59d34fa37b9b04f6f9e9926b3cece1a5",
|
||||
"sha256:b26104d6835d1f5e49452a26eb2ff87fe7090b89dfcaee5ea2212697e1e1d7ae"
|
||||
"sha256:e4f3620cfea4f83eedc95b24abd9cd56f3c4b146dd0177e83a21b4eb49e21e50",
|
||||
"sha256:fd7c7c74727ddcf00e9acd26bba8da604ffec95bf1c2144e67aff7a8b50e6cef"
|
||||
],
|
||||
"version": "==2019.3.9"
|
||||
"version": "==2019.9.11"
|
||||
},
|
||||
"chardet": {
|
||||
"hashes": [
|
||||
@ -32,19 +32,31 @@
|
||||
},
|
||||
"falcon": {
|
||||
"hashes": [
|
||||
"sha256:0a66b33458fab9c1e400a9be1a68056abda178eb02a8cb4b8f795e9df20b053b",
|
||||
"sha256:3981f609c0358a9fcdb25b0e7fab3d9e23019356fb429c635ce4133135ae1bc4"
|
||||
"sha256:18157af2a4fc3feedf2b5dcc6196f448639acf01c68bc33d4d5a04c3ef87f494",
|
||||
"sha256:24adcd2b29a8ffa9d552dc79638cd21736a3fb04eda7d102c6cebafdaadb88ad",
|
||||
"sha256:54f2cb4b687035b2a03206dbfc538055cc48b59a953187b0458aa1b574d47b53",
|
||||
"sha256:59d1e8c993b9a37ea06df9d72cf907a46cc8063b30717cdac2f34d1658b6f936",
|
||||
"sha256:733033ec80c896e30a43ab3e776856096836787197a44eb21022320a61311983",
|
||||
"sha256:74cf1d18207381c665b9e6292d65100ce146d958707793174b03869dc6e614f4",
|
||||
"sha256:95bf6ce986c1119aef12c9b348f4dee9c6dcc58391bdd0bc2b0bf353c2b15986",
|
||||
"sha256:9712975adcf8c6e12876239085ad757b8fdeba223d46d23daef82b47658f83a9",
|
||||
"sha256:a5ebb22a04c9cc65081938ee7651b4e3b4d2a28522ea8ec04c7bdd2b3e9e8cd8",
|
||||
"sha256:aa184895d1ad4573fbfaaf803563d02f019ebdf4790e41cc568a330607eae439",
|
||||
"sha256:e3782b7b92fefd46a6ad1fd8fe63fe6c6f1b7740a95ca56957f48d1aee34b357",
|
||||
"sha256:e9efa0791b5d9f9dd9689015ea6bce0a27fcd5ecbcd30e6d940bffa4f7f03389",
|
||||
"sha256:eea593cf466b9c126ce667f6d30503624ef24459f118c75594a69353b6c3d5fc",
|
||||
"sha256:f93351459f110b4c1ee28556aef9a791832df6f910bea7b3f616109d534df06b"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==1.4.1"
|
||||
"version": "==2.0.0"
|
||||
},
|
||||
"gunicorn": {
|
||||
"hashes": [
|
||||
"sha256:aa8e0b40b4157b36a5df5e599f45c9c76d6af43845ba3b3b0efe2c70473c2471",
|
||||
"sha256:fa2662097c66f920f53f70621c6c58ca4a3c4d3434205e608e121b5b3b71f4f3"
|
||||
"sha256:1904bb2b8a43658807108d59c3f3d56c2b6121a701161de0ddf9ad140073c626",
|
||||
"sha256:cd4a810dd51bf497552cf3f863b575dabd73d6ad6a91075b65936b151cbf4f9c"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==19.9.0"
|
||||
"version": "==20.0.4"
|
||||
},
|
||||
"idna": {
|
||||
"hashes": [
|
||||
@ -55,82 +67,72 @@
|
||||
},
|
||||
"psycopg2-binary": {
|
||||
"hashes": [
|
||||
"sha256:007ca0df127b1862fc010125bc4100b7a630efc6841047bd11afceadb4754611",
|
||||
"sha256:03c49e02adf0b4d68f422fdbd98f7a7c547beb27e99a75ed02298f85cb48406a",
|
||||
"sha256:0a1232cdd314e08848825edda06600455ad2a7adaa463ebfb12ece2d09f3370e",
|
||||
"sha256:131c80d0958c89273d9720b9adf9df1d7600bb3120e16019a7389ab15b079af5",
|
||||
"sha256:2de34cc3b775724623f86617d2601308083176a495f5b2efc2bbb0da154f483a",
|
||||
"sha256:2eddc31500f73544a2a54123d4c4b249c3c711d31e64deddb0890982ea37397a",
|
||||
"sha256:484f6c62bdc166ee0e5be3aa831120423bf399786d1f3b0304526c86180fbc0b",
|
||||
"sha256:4c2d9369ed40b4a44a8ccd6bc3a7db6272b8314812d2d1091f95c4c836d92e06",
|
||||
"sha256:70f570b5fa44413b9f30dbc053d17ef3ce6a4100147a10822f8662e58d473656",
|
||||
"sha256:7a2b5b095f3bd733aab101c89c0e1a3f0dfb4ebdc26f6374805c086ffe29d5b2",
|
||||
"sha256:804914a669186e2843c1f7fbe12b55aad1b36d40a28274abe6027deffad9433d",
|
||||
"sha256:8520c03172da18345d012949a53617a963e0191ccb3c666f23276d5326af27b5",
|
||||
"sha256:90da901fc33ea393fc644607e4a3916b509387e9339ec6ebc7bfded45b7a0ae9",
|
||||
"sha256:a582416ad123291a82c300d1d872bdc4136d69ad0b41d57dc5ca3df7ef8e3088",
|
||||
"sha256:ac8c5e20309f4989c296d62cac20ee456b69c41fd1bc03829e27de23b6fa9dd0",
|
||||
"sha256:b2cf82f55a619879f8557fdaae5cec7a294fac815e0087c4f67026fdf5259844",
|
||||
"sha256:b59d6f8cfca2983d8fdbe457bf95d2192f7b7efdb2b483bf5fa4e8981b04e8b2",
|
||||
"sha256:be08168197021d669b9964bd87628fa88f910b1be31e7010901070f2540c05fd",
|
||||
"sha256:be0f952f1c365061041bad16e27e224e29615d4eb1fb5b7e7760a1d3d12b90b6",
|
||||
"sha256:c1c9a33e46d7c12b9c96cf2d4349d783e3127163fd96254dcd44663cf0a1d438",
|
||||
"sha256:d18c89957ac57dd2a2724ecfe9a759912d776f96ecabba23acb9ecbf5c731035",
|
||||
"sha256:d7e7b0ff21f39433c50397e60bf0995d078802c591ca3b8d99857ea18a7496ee",
|
||||
"sha256:da0929b2bf0d1f365345e5eb940d8713c1d516312e010135b14402e2a3d2404d",
|
||||
"sha256:de24a4962e361c512d3e528ded6c7480eab24c655b8ca1f0b761d3b3650d2f07",
|
||||
"sha256:e45f93ff3f7dae2202248cf413a87aeb330821bf76998b3cf374eda2fc893dd7",
|
||||
"sha256:f046aeae1f7a845041b8661bb7a52449202b6c5d3fb59eb4724e7ca088811904",
|
||||
"sha256:f1dc2b7b2748084b890f5d05b65a47cd03188824890e9a60818721fd492249fb",
|
||||
"sha256:fcbe7cf3a786572b73d2cd5f34ed452a5f5fac47c9c9d1e0642c457a148f9f88"
|
||||
"sha256:040234f8a4a8dfd692662a8308d78f63f31a97e1c42d2480e5e6810c48966a29",
|
||||
"sha256:086f7e89ec85a6704db51f68f0dcae432eff9300809723a6e8782c41c2f48e03",
|
||||
"sha256:18ca813fdb17bc1db73fe61b196b05dd1ca2165b884dd5ec5568877cabf9b039",
|
||||
"sha256:19dc39616850342a2a6db70559af55b22955f86667b5f652f40c0e99253d9881",
|
||||
"sha256:2166e770cb98f02ed5ee2b0b569d40db26788e0bf2ec3ae1a0d864ea6f1d8309",
|
||||
"sha256:3a2522b1d9178575acee4adf8fd9f979f9c0449b00b4164bb63c3475ea6528ed",
|
||||
"sha256:3aa773580f85a28ffdf6f862e59cb5a3cc7ef6885121f2de3fca8d6ada4dbf3b",
|
||||
"sha256:3b5deaa3ee7180585a296af33e14c9b18c218d148e735c7accf78130765a47e3",
|
||||
"sha256:407af6d7e46593415f216c7f56ba087a9a42bd6dc2ecb86028760aa45b802bd7",
|
||||
"sha256:4c3c09fb674401f630626310bcaf6cd6285daf0d5e4c26d6e55ca26a2734e39b",
|
||||
"sha256:4c6717962247445b4f9e21c962ea61d2e884fc17df5ddf5e35863b016f8a1f03",
|
||||
"sha256:50446fae5681fc99f87e505d4e77c9407e683ab60c555ec302f9ac9bffa61103",
|
||||
"sha256:5057669b6a66aa9ca118a2a860159f0ee3acf837eda937bdd2a64f3431361a2d",
|
||||
"sha256:5dd90c5438b4f935c9d01fcbad3620253da89d19c1f5fca9158646407ed7df35",
|
||||
"sha256:659c815b5b8e2a55193ede2795c1e2349b8011497310bb936da7d4745652823b",
|
||||
"sha256:69b13fdf12878b10dc6003acc8d0abf3ad93e79813fd5f3812497c1c9fb9be49",
|
||||
"sha256:7a1cb80e35e1ccea3e11a48afe65d38744a0e0bde88795cc56a4d05b6e4f9d70",
|
||||
"sha256:7e6e3c52e6732c219c07bd97fff6c088f8df4dae3b79752ee3a817e6f32e177e",
|
||||
"sha256:7f42a8490c4fe854325504ce7a6e4796b207960dabb2cbafe3c3959cb00d1d7e",
|
||||
"sha256:84156313f258eafff716b2961644a4483a9be44a5d43551d554844d15d4d224e",
|
||||
"sha256:8578d6b8192e4c805e85f187bc530d0f52ba86c39172e61cd51f68fddd648103",
|
||||
"sha256:890167d5091279a27e2505ff0e1fb273f8c48c41d35c5b92adbf4af80e6b2ed6",
|
||||
"sha256:98e10634792ac0e9e7a92a76b4991b44c2325d3e7798270a808407355e7bb0a1",
|
||||
"sha256:9aadff9032e967865f9778485571e93908d27dab21d0fdfdec0ca779bb6f8ad9",
|
||||
"sha256:9f24f383a298a0c0f9b3113b982e21751a8ecde6615494a3f1470eb4a9d70e9e",
|
||||
"sha256:a73021b44813b5c84eda4a3af5826dd72356a900bac9bd9dd1f0f81ee1c22c2f",
|
||||
"sha256:afd96845e12638d2c44d213d4810a08f4dc4a563f9a98204b7428e567014b1cd",
|
||||
"sha256:b73ddf033d8cd4cc9dfed6324b1ad2a89ba52c410ef6877998422fcb9c23e3a8",
|
||||
"sha256:b8f490f5fad1767a1331df1259763b3bad7d7af12a75b950c2843ba319b2415f",
|
||||
"sha256:dbc5cd56fff1a6152ca59445178652756f4e509f672e49ccdf3d79c1043113a4",
|
||||
"sha256:eac8a3499754790187bb00574ab980df13e754777d346f85e0ff6df929bcd964",
|
||||
"sha256:eaed1c65f461a959284649e37b5051224f4db6ebdc84e40b5e65f2986f101a08"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==2.8.2"
|
||||
},
|
||||
"python-mimeparse": {
|
||||
"hashes": [
|
||||
"sha256:76e4b03d700a641fd7761d3cd4fdbbdcd787eade1ebfac43f877016328334f78",
|
||||
"sha256:a295f03ff20341491bfe4717a39cd0a8cc9afad619ba44b77e86b0ab8a2b8282"
|
||||
],
|
||||
"version": "==1.6.0"
|
||||
"version": "==2.8.4"
|
||||
},
|
||||
"requests": {
|
||||
"hashes": [
|
||||
"sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e",
|
||||
"sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b"
|
||||
"sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4",
|
||||
"sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==2.21.0"
|
||||
},
|
||||
"six": {
|
||||
"hashes": [
|
||||
"sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
|
||||
"sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"
|
||||
],
|
||||
"version": "==1.12.0"
|
||||
"version": "==2.22.0"
|
||||
},
|
||||
"urllib3": {
|
||||
"hashes": [
|
||||
"sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39",
|
||||
"sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22"
|
||||
"sha256:a8a318824cc77d1fd4b2bec2ded92646630d7fe8619497b142c84a9e6f5a7293",
|
||||
"sha256:f3c5fd51747d450d4dcf6f923c81f78f811aab8205fda64b0aba34a4e48b0745"
|
||||
],
|
||||
"version": "==1.24.1"
|
||||
"version": "==1.25.7"
|
||||
}
|
||||
},
|
||||
"develop": {
|
||||
"atomicwrites": {
|
||||
"appdirs": {
|
||||
"hashes": [
|
||||
"sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4",
|
||||
"sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6"
|
||||
"sha256:9e5896d1372858f8dd3344faf4e5014d21849c756c8d5701f78f8a103b372d92",
|
||||
"sha256:d8b24664561d0d34ddfaec54636d502d7cea6e29c3eaf68f3df6180863e2166e"
|
||||
],
|
||||
"version": "==1.3.0"
|
||||
"version": "==1.4.3"
|
||||
},
|
||||
"attrs": {
|
||||
"hashes": [
|
||||
"sha256:69c0dbf2ed392de1cb5ec704444b08a5ef81680a61cb899dc08127123af36a79",
|
||||
"sha256:f0b870f674851ecbfbbbd364d6b5cbdff9dcedbc7f3f5e18a6891057f21fe399"
|
||||
"sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c",
|
||||
"sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"
|
||||
],
|
||||
"version": "==19.1.0"
|
||||
"version": "==19.3.0"
|
||||
},
|
||||
"backcall": {
|
||||
"hashes": [
|
||||
@ -139,12 +141,27 @@
|
||||
],
|
||||
"version": "==0.1.0"
|
||||
},
|
||||
"black": {
|
||||
"hashes": [
|
||||
"sha256:1b30e59be925fafc1ee4565e5e08abef6b03fe455102883820fe5ee2e4734e0b",
|
||||
"sha256:c2edb73a08e9e0e6f65a0e6af18b059b8b1cdd5bef997d7a0b181df93dc81539"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==19.10b0"
|
||||
},
|
||||
"click": {
|
||||
"hashes": [
|
||||
"sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13",
|
||||
"sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7"
|
||||
],
|
||||
"version": "==7.0"
|
||||
},
|
||||
"decorator": {
|
||||
"hashes": [
|
||||
"sha256:86156361c50488b84a3f148056ea716ca587df2f0de1d34750d35c21312725de",
|
||||
"sha256:f069f3a01830ca754ba5258fde2278454a0b5b79e0d7f5c13b3b97e57d4acff6"
|
||||
"sha256:54c38050039232e1db4ad7375cfce6748d7b41c29e95a081c8a6d2c30364a2ce",
|
||||
"sha256:5d19b92a3c8f7f101c8dd86afd86b0f061a8ce4540ab8cd401fa2542756bce6d"
|
||||
],
|
||||
"version": "==4.4.0"
|
||||
"version": "==4.4.1"
|
||||
},
|
||||
"entrypoints": {
|
||||
"hashes": [
|
||||
@ -155,19 +172,19 @@
|
||||
},
|
||||
"flake8": {
|
||||
"hashes": [
|
||||
"sha256:859996073f341f2670741b51ec1e67a01da142831aa1fdc6242dbf88dffbe661",
|
||||
"sha256:a796a115208f5c03b18f332f7c11729812c8c3ded6c46319c59b53efd3819da8"
|
||||
"sha256:45681a117ecc81e870cbf1262835ae4af5e7a8b08e40b944a8a6e6b895914cfb",
|
||||
"sha256:49356e766643ad15072a789a20915d3c91dc89fd313ccd71802303fd67e4deca"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==3.7.7"
|
||||
"version": "==3.7.9"
|
||||
},
|
||||
"ipython": {
|
||||
"hashes": [
|
||||
"sha256:b038baa489c38f6d853a3cfc4c635b0cda66f2864d136fe8f40c1a6e334e2a6b",
|
||||
"sha256:f5102c1cd67e399ec8ea66bcebe6e3968ea25a8977e53f012963e5affeb1fe38"
|
||||
"sha256:dfd303b270b7b5232b3d08bd30ec6fd685d8a58cabd54055e3d69d8f029f7280",
|
||||
"sha256:ed7ebe1cba899c1c3ccad6f7f1c2d2369464cc77dba8eebc65e2043e19cda995"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==7.4.0"
|
||||
"version": "==7.9.0"
|
||||
},
|
||||
"ipython-genutils": {
|
||||
"hashes": [
|
||||
@ -176,12 +193,20 @@
|
||||
],
|
||||
"version": "==0.2.0"
|
||||
},
|
||||
"isort": {
|
||||
"hashes": [
|
||||
"sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1",
|
||||
"sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==4.3.21"
|
||||
},
|
||||
"jedi": {
|
||||
"hashes": [
|
||||
"sha256:2bb0603e3506f708e792c7f4ad8fc2a7a9d9c2d292a358fbbd58da531695595b",
|
||||
"sha256:2c6bcd9545c7d6440951b12b44d373479bf18123a401a52025cf98563fbd826c"
|
||||
"sha256:786b6c3d80e2f06fd77162a07fed81b8baa22dde5d62896a790a331d6ac21a27",
|
||||
"sha256:ba859c74fa3c966a22f2aeebe1b74ee27e2a462f56d3f5f7ca4a59af61bfe42e"
|
||||
],
|
||||
"version": "==0.13.3"
|
||||
"version": "==0.15.1"
|
||||
},
|
||||
"mccabe": {
|
||||
"hashes": [
|
||||
@ -192,18 +217,30 @@
|
||||
},
|
||||
"more-itertools": {
|
||||
"hashes": [
|
||||
"sha256:2112d2ca570bb7c3e53ea1a35cd5df42bb0fd10c45f0fb97178679c3c03d64c7",
|
||||
"sha256:c3e4748ba1aad8dba30a4886b0b1a2004f9a863837b8654e7059eebf727afa5a"
|
||||
"sha256:409cd48d4db7052af495b09dec721011634af3753ae1ef92d2b32f73a745f832",
|
||||
"sha256:92b8c4b06dac4f0611c0729b2f2ede52b2e1bac1ab48f089c7ddc12e26bb60c4"
|
||||
],
|
||||
"markers": "python_version > '2.7'",
|
||||
"version": "==7.0.0"
|
||||
"version": "==7.2.0"
|
||||
},
|
||||
"packaging": {
|
||||
"hashes": [
|
||||
"sha256:28b924174df7a2fa32c1953825ff29c61e2f5e082343165438812f00d3a7fc47",
|
||||
"sha256:d9551545c6d761f3def1677baf08ab2a3ca17c56879e70fecba2fc4dde4ed108"
|
||||
],
|
||||
"version": "==19.2"
|
||||
},
|
||||
"parso": {
|
||||
"hashes": [
|
||||
"sha256:17cc2d7a945eb42c3569d4564cdf49bde221bc2b552af3eca9c1aad517dcdd33",
|
||||
"sha256:2e9574cb12e7112a87253e14e2c380ce312060269d04bd018478a3c92ea9a376"
|
||||
"sha256:63854233e1fadb5da97f2744b6b24346d2750b85965e7e399bec1620232797dc",
|
||||
"sha256:666b0ee4a7a1220f65d367617f2cd3ffddff3e205f3f16a0284df30e774c2a9c"
|
||||
],
|
||||
"version": "==0.4.0"
|
||||
"version": "==0.5.1"
|
||||
},
|
||||
"pathspec": {
|
||||
"hashes": [
|
||||
"sha256:e285ccc8b0785beadd4c18e5708b12bb8fcf529a1e61215b3feff1d1e559ea5c"
|
||||
],
|
||||
"version": "==0.6.0"
|
||||
},
|
||||
"pexpect": {
|
||||
"hashes": [
|
||||
@ -222,18 +259,18 @@
|
||||
},
|
||||
"pluggy": {
|
||||
"hashes": [
|
||||
"sha256:19ecf9ce9db2fce065a7a0586e07cfb4ac8614fe96edf628a264b1c70116cf8f",
|
||||
"sha256:84d306a647cc805219916e62aab89caa97a33a1dd8c342e87a37f91073cd4746"
|
||||
"sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0",
|
||||
"sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"
|
||||
],
|
||||
"version": "==0.9.0"
|
||||
"version": "==0.13.1"
|
||||
},
|
||||
"prompt-toolkit": {
|
||||
"hashes": [
|
||||
"sha256:11adf3389a996a6d45cc277580d0d53e8a5afd281d0c9ec71b28e6f121463780",
|
||||
"sha256:2519ad1d8038fd5fc8e770362237ad0364d16a7650fb5724af6997ed5515e3c1",
|
||||
"sha256:977c6583ae813a37dc1c2e1b715892461fcbdaa57f6fc62f33a528c4886c8f55"
|
||||
"sha256:46642344ce457641f28fc9d1c9ca939b63dadf8df128b86f1b9860e59c73a5e4",
|
||||
"sha256:e7f8af9e3d70f514373bf41aa51bc33af12a6db3f71461ea47fea985defb2c31",
|
||||
"sha256:f15af68f66e664eaa559d4ac8a928111eebd5feda0c11738b5998045224829db"
|
||||
],
|
||||
"version": "==2.0.9"
|
||||
"version": "==2.0.10"
|
||||
},
|
||||
"ptyprocess": {
|
||||
"hashes": [
|
||||
@ -265,32 +302,89 @@
|
||||
},
|
||||
"pygments": {
|
||||
"hashes": [
|
||||
"sha256:5ffada19f6203563680669ee7f53b64dabbeb100eb51b61996085e99c03b284a",
|
||||
"sha256:e8218dd399a61674745138520d0d4cf2621d7e032439341bc3f647bff125818d"
|
||||
"sha256:83ec6c6133ca6b529b7ff5aa826328fd14b5bb02a58c37f4f06384e96a0f94ab",
|
||||
"sha256:b7949de3d396836085fea596998b135a22610bbcc4f2abfe9e448e44cbc58388"
|
||||
],
|
||||
"version": "==2.3.1"
|
||||
"version": "==2.5.1"
|
||||
},
|
||||
"pyparsing": {
|
||||
"hashes": [
|
||||
"sha256:20f995ecd72f2a1f4bf6b072b63b22e2eb457836601e76d6e5dfcd75436acc1f",
|
||||
"sha256:4ca62001be367f01bd3e92ecbb79070272a9d4964dce6a48a82ff0b8bc7e683a"
|
||||
],
|
||||
"version": "==2.4.5"
|
||||
},
|
||||
"pytest": {
|
||||
"hashes": [
|
||||
"sha256:13c5e9fb5ec5179995e9357111ab089af350d788cbc944c628f3cde72285809b",
|
||||
"sha256:f21d2f1fb8200830dcbb5d8ec466a9c9120e20d8b53c7585d180125cce1d297a"
|
||||
"sha256:63344a2e3bce2e4d522fd62b4fdebb647c019f1f9e4ca075debbd13219db4418",
|
||||
"sha256:f67403f33b2b1d25a6756184077394167fe5e2f9d8bdaab30707d19ccec35427"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==4.4.0"
|
||||
"version": "==5.3.1"
|
||||
},
|
||||
"regex": {
|
||||
"hashes": [
|
||||
"sha256:15454b37c5a278f46f7aa2d9339bda450c300617ca2fca6558d05d870245edc7",
|
||||
"sha256:1ad40708c255943a227e778b022c6497c129ad614bb7a2a2f916e12e8a359ee7",
|
||||
"sha256:5e00f65cc507d13ab4dfa92c1232d004fa202c1d43a32a13940ab8a5afe2fb96",
|
||||
"sha256:604dc563a02a74d70ae1f55208ddc9bfb6d9f470f6d1a5054c4bd5ae58744ab1",
|
||||
"sha256:720e34a539a76a1fedcebe4397290604cc2bdf6f81eca44adb9fb2ea071c0c69",
|
||||
"sha256:7caf47e4a9ac6ef08cabd3442cc4ca3386db141fb3c8b2a7e202d0470028e910",
|
||||
"sha256:7faf534c1841c09d8fefa60ccde7b9903c9b528853ecf41628689793290ca143",
|
||||
"sha256:b4e0406d822aa4993ac45072a584d57aa4931cf8288b5455bbf30c1d59dbad59",
|
||||
"sha256:c31eaf28c6fe75ea329add0022efeed249e37861c19681960f99bbc7db981fb2",
|
||||
"sha256:c7393597191fc2043c744db021643549061e12abe0b3ff5c429d806de7b93b66",
|
||||
"sha256:d2b302f8cdd82c8f48e9de749d1d17f85ce9a0f082880b9a4859f66b07037dc6",
|
||||
"sha256:e3d8dd0ec0ea280cf89026b0898971f5750a7bd92cb62c51af5a52abd020054a",
|
||||
"sha256:ec032cbfed59bd5a4b8eab943c310acfaaa81394e14f44454ad5c9eba4f24a74"
|
||||
],
|
||||
"version": "==2019.11.1"
|
||||
},
|
||||
"six": {
|
||||
"hashes": [
|
||||
"sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c",
|
||||
"sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73"
|
||||
"sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd",
|
||||
"sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66"
|
||||
],
|
||||
"version": "==1.12.0"
|
||||
"version": "==1.13.0"
|
||||
},
|
||||
"toml": {
|
||||
"hashes": [
|
||||
"sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c",
|
||||
"sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e"
|
||||
],
|
||||
"version": "==0.10.0"
|
||||
},
|
||||
"traitlets": {
|
||||
"hashes": [
|
||||
"sha256:9c4bd2d267b7153df9152698efb1050a5d84982d3384a37b2c1f7723ba3e7835",
|
||||
"sha256:c6cb5e6f57c5a9bdaa40fa71ce7b4af30298fbab9ece9815b5d995ab6217c7d9"
|
||||
"sha256:70b4c6a1d9019d7b4f6846832288f86998aa3b9207c6821f3578a6a6a467fe44",
|
||||
"sha256:d023ee369ddd2763310e4c3eae1ff649689440d4ae59d7485eb4cfbbe3e359f7"
|
||||
],
|
||||
"version": "==4.3.2"
|
||||
"version": "==4.3.3"
|
||||
},
|
||||
"typed-ast": {
|
||||
"hashes": [
|
||||
"sha256:1170afa46a3799e18b4c977777ce137bb53c7485379d9706af8a59f2ea1aa161",
|
||||
"sha256:18511a0b3e7922276346bcb47e2ef9f38fb90fd31cb9223eed42c85d1312344e",
|
||||
"sha256:262c247a82d005e43b5b7f69aff746370538e176131c32dda9cb0f324d27141e",
|
||||
"sha256:2b907eb046d049bcd9892e3076c7a6456c93a25bebfe554e931620c90e6a25b0",
|
||||
"sha256:354c16e5babd09f5cb0ee000d54cfa38401d8b8891eefa878ac772f827181a3c",
|
||||
"sha256:48e5b1e71f25cfdef98b013263a88d7145879fbb2d5185f2a0c79fa7ebbeae47",
|
||||
"sha256:4e0b70c6fc4d010f8107726af5fd37921b666f5b31d9331f0bd24ad9a088e631",
|
||||
"sha256:630968c5cdee51a11c05a30453f8cd65e0cc1d2ad0d9192819df9978984529f4",
|
||||
"sha256:66480f95b8167c9c5c5c87f32cf437d585937970f3fc24386f313a4c97b44e34",
|
||||
"sha256:71211d26ffd12d63a83e079ff258ac9d56a1376a25bc80b1cdcdf601b855b90b",
|
||||
"sha256:7954560051331d003b4e2b3eb822d9dd2e376fa4f6d98fee32f452f52dd6ebb2",
|
||||
"sha256:838997f4310012cf2e1ad3803bce2f3402e9ffb71ded61b5ee22617b3a7f6b6e",
|
||||
"sha256:95bd11af7eafc16e829af2d3df510cecfd4387f6453355188342c3e79a2ec87a",
|
||||
"sha256:bc6c7d3fa1325a0c6613512a093bc2a2a15aeec350451cbdf9e1d4bffe3e3233",
|
||||
"sha256:cc34a6f5b426748a507dd5d1de4c1978f2eb5626d51326e43280941206c209e1",
|
||||
"sha256:d755f03c1e4a51e9b24d899561fec4ccaf51f210d52abdf8c07ee2849b212a36",
|
||||
"sha256:d7c45933b1bdfaf9f36c579671fec15d25b06c8398f113dab64c18ed1adda01d",
|
||||
"sha256:d896919306dd0aa22d0132f62a1b78d11aaf4c9fc5b3410d3c666b818191630a",
|
||||
"sha256:fdc1c9bbf79510b76408840e009ed65958feba92a88833cdceecff93ae8fff66",
|
||||
"sha256:ffde2fbfad571af120fcbfbbc61c72469e72f550d676c3342492a9dfdefb8f12"
|
||||
],
|
||||
"version": "==1.4.0"
|
||||
},
|
||||
"wcwidth": {
|
||||
"hashes": [
|
||||
|
@ -1,4 +1,4 @@
|
||||
# DSpace Statistics API [](https://travis-ci.org/ilri/dspace-statistics-api) [](https://builds.sr.ht/~alanorth/dspace-statistics-api?)
|
||||
# DSpace Statistics API [](https://travis-ci.com/ilri/dspace-statistics-api) [](https://builds.sr.ht/~alanorth/dspace-statistics-api?)
|
||||
DSpace stores item view and download events in a Solr "statistics" core. This information is available for use in the various DSpace user interfaces, but is not exposed externally via any APIs. The DSpace 4/5 [REST API](https://wiki.duraspace.org/display/DSDOC5x/REST+API), for example, only exposes information about communities, collections, item metadata, and bitstreams.
|
||||
|
||||
This project contains an indexer and a [Falcon-based](https://falcon.readthedocs.io/) web application to make the statistics available via a simple REST API. You can read more about the Solr queries used to gather the item view and download statistics on the [DSpace wiki](https://wiki.duraspace.org/display/DSPACE/Solr).
|
||||
|
@ -1,12 +1,13 @@
|
||||
from .database import DatabaseManager
|
||||
import falcon
|
||||
|
||||
from .database import DatabaseManager
|
||||
|
||||
|
||||
class RootResource:
|
||||
def on_get(self, req, resp):
|
||||
resp.status = falcon.HTTP_200
|
||||
resp.content_type = 'text/html'
|
||||
with open('dspace_statistics_api/docs/index.html', 'r') as f:
|
||||
resp.content_type = "text/html"
|
||||
with open("dspace_statistics_api/docs/index.html", "r") as f:
|
||||
resp.body = f.read()
|
||||
|
||||
|
||||
@ -14,8 +15,8 @@ class AllItemsResource:
|
||||
def on_get(self, req, resp):
|
||||
"""Handles GET requests"""
|
||||
# Return HTTPBadRequest if id parameter is not present and valid
|
||||
limit = req.get_param_as_int("limit", min=0, max=100) or 100
|
||||
page = req.get_param_as_int("page", min=0) or 0
|
||||
limit = req.get_param_as_int("limit", min_value=0, max_value=100) or 100
|
||||
page = req.get_param_as_int("page", min_value=0) or 0
|
||||
offset = limit * page
|
||||
|
||||
with DatabaseManager() as db:
|
||||
@ -23,24 +24,34 @@ class AllItemsResource:
|
||||
|
||||
with db.cursor() as cursor:
|
||||
# get total number of items so we can estimate the pages
|
||||
cursor.execute('SELECT COUNT(id) FROM items')
|
||||
cursor.execute("SELECT COUNT(id) FROM items")
|
||||
pages = round(cursor.fetchone()[0] / limit)
|
||||
|
||||
# get statistics, ordered by id, and use limit and offset to page through results
|
||||
cursor.execute('SELECT id, views, downloads FROM items ORDER BY id ASC LIMIT {} OFFSET {}'.format(limit, offset))
|
||||
cursor.execute(
|
||||
"SELECT id, views, downloads FROM items ORDER BY id ASC LIMIT {} OFFSET {}".format(
|
||||
limit, offset
|
||||
)
|
||||
)
|
||||
|
||||
# create a list to hold dicts of item stats
|
||||
statistics = list()
|
||||
|
||||
# iterate over results and build statistics object
|
||||
for item in cursor:
|
||||
statistics.append({'id': item['id'], 'views': item['views'], 'downloads': item['downloads']})
|
||||
statistics.append(
|
||||
{
|
||||
"id": item["id"],
|
||||
"views": item["views"],
|
||||
"downloads": item["downloads"],
|
||||
}
|
||||
)
|
||||
|
||||
message = {
|
||||
'currentPage': page,
|
||||
'totalPages': pages,
|
||||
'limit': limit,
|
||||
'statistics': statistics
|
||||
"currentPage": page,
|
||||
"totalPages": pages,
|
||||
"limit": limit,
|
||||
"statistics": statistics,
|
||||
}
|
||||
|
||||
resp.media = message
|
||||
@ -55,27 +66,31 @@ class ItemResource:
|
||||
|
||||
with db.cursor() as cursor:
|
||||
cursor = db.cursor()
|
||||
cursor.execute('SELECT views, downloads FROM items WHERE id={}'.format(item_id))
|
||||
cursor.execute(
|
||||
"SELECT views, downloads FROM items WHERE id={}".format(item_id)
|
||||
)
|
||||
if cursor.rowcount == 0:
|
||||
raise falcon.HTTPNotFound(
|
||||
title='Item not found',
|
||||
description='The item with id "{}" was not found.'.format(item_id)
|
||||
title="Item not found",
|
||||
description='The item with id "{}" was not found.'.format(
|
||||
item_id
|
||||
),
|
||||
)
|
||||
else:
|
||||
results = cursor.fetchone()
|
||||
|
||||
statistics = {
|
||||
'id': item_id,
|
||||
'views': results['views'],
|
||||
'downloads': results['downloads']
|
||||
"id": item_id,
|
||||
"views": results["views"],
|
||||
"downloads": results["downloads"],
|
||||
}
|
||||
|
||||
resp.media = statistics
|
||||
|
||||
|
||||
api = application = falcon.API()
|
||||
api.add_route('/', RootResource())
|
||||
api.add_route('/items', AllItemsResource())
|
||||
api.add_route('/item/{item_id:int}', ItemResource())
|
||||
api.add_route("/", RootResource())
|
||||
api.add_route("/items", AllItemsResource())
|
||||
api.add_route("/item/{item_id:int}", ItemResource())
|
||||
|
||||
# vim: set sw=4 ts=4 expandtab:
|
||||
|
@ -1,12 +1,12 @@
|
||||
import os
|
||||
|
||||
# Check if Solr connection information was provided in the environment
|
||||
SOLR_SERVER = os.environ.get('SOLR_SERVER', 'http://localhost:8080/solr')
|
||||
SOLR_SERVER = os.environ.get("SOLR_SERVER", "http://localhost:8080/solr")
|
||||
|
||||
DATABASE_NAME = os.environ.get('DATABASE_NAME', 'dspacestatistics')
|
||||
DATABASE_USER = os.environ.get('DATABASE_USER', 'dspacestatistics')
|
||||
DATABASE_PASS = os.environ.get('DATABASE_PASS', 'dspacestatistics')
|
||||
DATABASE_HOST = os.environ.get('DATABASE_HOST', 'localhost')
|
||||
DATABASE_PORT = os.environ.get('DATABASE_PORT', '5432')
|
||||
DATABASE_NAME = os.environ.get("DATABASE_NAME", "dspacestatistics")
|
||||
DATABASE_USER = os.environ.get("DATABASE_USER", "dspacestatistics")
|
||||
DATABASE_PASS = os.environ.get("DATABASE_PASS", "dspacestatistics")
|
||||
DATABASE_HOST = os.environ.get("DATABASE_HOST", "localhost")
|
||||
DATABASE_PORT = os.environ.get("DATABASE_PORT", "5432")
|
||||
|
||||
# vim: set sw=4 ts=4 expandtab:
|
||||
|
@ -1,25 +1,32 @@
|
||||
from .config import DATABASE_NAME
|
||||
from .config import DATABASE_USER
|
||||
from .config import DATABASE_PASS
|
||||
from .config import DATABASE_HOST
|
||||
from .config import DATABASE_PORT
|
||||
import falcon
|
||||
import psycopg2
|
||||
import psycopg2.extras
|
||||
|
||||
from .config import (
|
||||
DATABASE_HOST,
|
||||
DATABASE_NAME,
|
||||
DATABASE_PASS,
|
||||
DATABASE_PORT,
|
||||
DATABASE_USER,
|
||||
)
|
||||
|
||||
class DatabaseManager():
|
||||
'''Manage database connection.'''
|
||||
|
||||
class DatabaseManager:
|
||||
"""Manage database connection."""
|
||||
|
||||
def __init__(self):
|
||||
self._connection_uri = 'dbname={} user={} password={} host={} port={}'.format(DATABASE_NAME, DATABASE_USER, DATABASE_PASS, DATABASE_HOST, DATABASE_PORT)
|
||||
self._connection_uri = "dbname={} user={} password={} host={} port={}".format(
|
||||
DATABASE_NAME, DATABASE_USER, DATABASE_PASS, DATABASE_HOST, DATABASE_PORT
|
||||
)
|
||||
|
||||
def __enter__(self):
|
||||
try:
|
||||
self._connection = psycopg2.connect(self._connection_uri, cursor_factory=psycopg2.extras.DictCursor)
|
||||
self._connection = psycopg2.connect(
|
||||
self._connection_uri, cursor_factory=psycopg2.extras.DictCursor
|
||||
)
|
||||
except psycopg2.OperationalError:
|
||||
title = '500 Internal Server Error'
|
||||
description = 'Could not connect to database'
|
||||
title = "500 Internal Server Error"
|
||||
description = "Could not connect to database"
|
||||
raise falcon.HTTPInternalServerError(title, description)
|
||||
|
||||
return self._connection
|
||||
@ -27,4 +34,5 @@ class DatabaseManager():
|
||||
def __exit__(self, exc_type, exc_value, exc_traceback):
|
||||
self._connection.close()
|
||||
|
||||
|
||||
# vim: set sw=4 ts=4 expandtab:
|
||||
|
@ -29,12 +29,13 @@
|
||||
# See: https://solrclient.readthedocs.io/en/latest/SolrClient.html
|
||||
# See: https://wiki.duraspace.org/display/DSPACE/Solr
|
||||
|
||||
import re
|
||||
|
||||
import psycopg2.extras
|
||||
import requests
|
||||
|
||||
from .config import SOLR_SERVER
|
||||
from .database import DatabaseManager
|
||||
import json
|
||||
import psycopg2.extras
|
||||
import re
|
||||
import requests
|
||||
|
||||
|
||||
# Enumerate the cores in Solr to determine if statistics have been sharded into
|
||||
@ -44,11 +45,8 @@ def get_statistics_shards():
|
||||
statistics_core_years = []
|
||||
|
||||
# URL for Solr status to check active cores
|
||||
solr_query_params = {
|
||||
'action': 'STATUS',
|
||||
'wt': 'json'
|
||||
}
|
||||
solr_url = SOLR_SERVER + '/admin/cores'
|
||||
solr_query_params = {"action": "STATUS", "wt": "json"}
|
||||
solr_url = SOLR_SERVER + "/admin/cores"
|
||||
res = requests.get(solr_url, params=solr_query_params)
|
||||
|
||||
if res.status_code == requests.codes.ok:
|
||||
@ -56,9 +54,9 @@ def get_statistics_shards():
|
||||
|
||||
# Iterate over active cores from Solr's STATUS response (cores are in
|
||||
# the status array of this response).
|
||||
for core in data['status']:
|
||||
for core in data["status"]:
|
||||
# Pattern to match, for example: statistics-2018
|
||||
pattern = re.compile('^statistics-[0-9]{4}$')
|
||||
pattern = re.compile("^statistics-[0-9]{4}$")
|
||||
|
||||
if not pattern.match(core):
|
||||
continue
|
||||
@ -72,13 +70,13 @@ def get_statistics_shards():
|
||||
|
||||
if len(statistics_core_years) > 0:
|
||||
# Begin building a string of shards starting with the default one
|
||||
shards = '{}/statistics'.format(SOLR_SERVER)
|
||||
shards = "{}/statistics".format(SOLR_SERVER)
|
||||
|
||||
for core in statistics_core_years:
|
||||
# Create a comma-separated list of shards to pass to our Solr query
|
||||
#
|
||||
# See: https://wiki.apache.org/solr/DistributedSearch
|
||||
shards += ',{}/{}'.format(SOLR_SERVER, core)
|
||||
shards += ",{}/{}".format(SOLR_SERVER, core)
|
||||
|
||||
# Return the string of shards, which may actually be empty. Solr doesn't
|
||||
# seem to mind if the shards query parameter is empty and I haven't seen
|
||||
@ -94,30 +92,32 @@ def index_views():
|
||||
#
|
||||
# see: https://lucene.apache.org/solr/guide/6_6/the-stats-component.html
|
||||
solr_query_params = {
|
||||
'q': 'type:2',
|
||||
'fq': 'isBot:false AND statistics_type:view',
|
||||
'facet': 'true',
|
||||
'facet.field': 'id',
|
||||
'facet.mincount': 1,
|
||||
'facet.limit': 1,
|
||||
'facet.offset': 0,
|
||||
'stats': 'true',
|
||||
'stats.field': 'id',
|
||||
'stats.calcdistinct': 'true',
|
||||
'shards': shards,
|
||||
'rows': 0,
|
||||
'wt': 'json'
|
||||
"q": "type:2",
|
||||
"fq": "isBot:false AND statistics_type:view",
|
||||
"facet": "true",
|
||||
"facet.field": "id",
|
||||
"facet.mincount": 1,
|
||||
"facet.limit": 1,
|
||||
"facet.offset": 0,
|
||||
"stats": "true",
|
||||
"stats.field": "id",
|
||||
"stats.calcdistinct": "true",
|
||||
"shards": shards,
|
||||
"rows": 0,
|
||||
"wt": "json",
|
||||
}
|
||||
|
||||
solr_url = SOLR_SERVER + '/statistics/select'
|
||||
solr_url = SOLR_SERVER + "/statistics/select"
|
||||
|
||||
res = requests.get(solr_url, params=solr_query_params)
|
||||
|
||||
try:
|
||||
# get total number of distinct facets (countDistinct)
|
||||
results_totalNumFacets = res.json()['stats']['stats_fields']['id']['countDistinct']
|
||||
results_totalNumFacets = res.json()["stats"]["stats_fields"]["id"][
|
||||
"countDistinct"
|
||||
]
|
||||
except TypeError:
|
||||
print('No item views to index, exiting.')
|
||||
print("No item views to index, exiting.")
|
||||
|
||||
exit(0)
|
||||
|
||||
@ -133,35 +133,39 @@ def index_views():
|
||||
|
||||
while results_current_page <= results_num_pages:
|
||||
# "pages" are zero based, but one based is more human readable
|
||||
print('Indexing item views (page {} of {})'.format(results_current_page + 1, results_num_pages + 1))
|
||||
print(
|
||||
"Indexing item views (page {} of {})".format(
|
||||
results_current_page + 1, results_num_pages + 1
|
||||
)
|
||||
)
|
||||
|
||||
solr_query_params = {
|
||||
'q': 'type:2',
|
||||
'fq': 'isBot:false AND statistics_type:view',
|
||||
'facet': 'true',
|
||||
'facet.field': 'id',
|
||||
'facet.mincount': 1,
|
||||
'facet.limit': results_per_page,
|
||||
'facet.offset': results_current_page * results_per_page,
|
||||
'shards': shards,
|
||||
'rows': 0,
|
||||
'wt': 'json',
|
||||
'json.nl': 'map' # return facets as a dict instead of a flat list
|
||||
"q": "type:2",
|
||||
"fq": "isBot:false AND statistics_type:view",
|
||||
"facet": "true",
|
||||
"facet.field": "id",
|
||||
"facet.mincount": 1,
|
||||
"facet.limit": results_per_page,
|
||||
"facet.offset": results_current_page * results_per_page,
|
||||
"shards": shards,
|
||||
"rows": 0,
|
||||
"wt": "json",
|
||||
"json.nl": "map", # return facets as a dict instead of a flat list
|
||||
}
|
||||
|
||||
solr_url = SOLR_SERVER + '/statistics/select'
|
||||
solr_url = SOLR_SERVER + "/statistics/select"
|
||||
|
||||
res = requests.get(solr_url, params=solr_query_params)
|
||||
|
||||
# Solr returns facets as a dict of dicts (see json.nl parameter)
|
||||
views = res.json()['facet_counts']['facet_fields']
|
||||
views = res.json()["facet_counts"]["facet_fields"]
|
||||
# iterate over the 'id' dict and get the item ids and views
|
||||
for item_id, item_views in views['id'].items():
|
||||
for item_id, item_views in views["id"].items():
|
||||
data.append((item_id, item_views))
|
||||
|
||||
# do a batch insert of values from the current "page" of results
|
||||
sql = 'INSERT INTO items(id, views) VALUES %s ON CONFLICT(id) DO UPDATE SET views=excluded.views'
|
||||
psycopg2.extras.execute_values(cursor, sql, data, template='(%s, %s)')
|
||||
sql = "INSERT INTO items(id, views) VALUES %s ON CONFLICT(id) DO UPDATE SET views=excluded.views"
|
||||
psycopg2.extras.execute_values(cursor, sql, data, template="(%s, %s)")
|
||||
db.commit()
|
||||
|
||||
# clear all items from the list so we can populate it with the next batch
|
||||
@ -172,31 +176,33 @@ def index_views():
|
||||
|
||||
def index_downloads():
|
||||
# get the total number of distinct facets for items with at least 1 download
|
||||
solr_query_params= {
|
||||
'q': 'type:0',
|
||||
'fq': 'isBot:false AND statistics_type:view AND bundleName:ORIGINAL',
|
||||
'facet': 'true',
|
||||
'facet.field': 'owningItem',
|
||||
'facet.mincount': 1,
|
||||
'facet.limit': 1,
|
||||
'facet.offset': 0,
|
||||
'stats': 'true',
|
||||
'stats.field': 'owningItem',
|
||||
'stats.calcdistinct': 'true',
|
||||
'shards': shards,
|
||||
'rows': 0,
|
||||
'wt': 'json'
|
||||
solr_query_params = {
|
||||
"q": "type:0",
|
||||
"fq": "isBot:false AND statistics_type:view AND bundleName:ORIGINAL",
|
||||
"facet": "true",
|
||||
"facet.field": "owningItem",
|
||||
"facet.mincount": 1,
|
||||
"facet.limit": 1,
|
||||
"facet.offset": 0,
|
||||
"stats": "true",
|
||||
"stats.field": "owningItem",
|
||||
"stats.calcdistinct": "true",
|
||||
"shards": shards,
|
||||
"rows": 0,
|
||||
"wt": "json",
|
||||
}
|
||||
|
||||
solr_url = SOLR_SERVER + '/statistics/select'
|
||||
solr_url = SOLR_SERVER + "/statistics/select"
|
||||
|
||||
res = requests.get(solr_url, params=solr_query_params)
|
||||
|
||||
try:
|
||||
# get total number of distinct facets (countDistinct)
|
||||
results_totalNumFacets = res.json()['stats']['stats_fields']['owningItem']['countDistinct']
|
||||
results_totalNumFacets = res.json()["stats"]["stats_fields"]["owningItem"][
|
||||
"countDistinct"
|
||||
]
|
||||
except TypeError:
|
||||
print('No item downloads to index, exiting.')
|
||||
print("No item downloads to index, exiting.")
|
||||
|
||||
exit(0)
|
||||
|
||||
@ -212,35 +218,39 @@ def index_downloads():
|
||||
|
||||
while results_current_page <= results_num_pages:
|
||||
# "pages" are zero based, but one based is more human readable
|
||||
print('Indexing item downloads (page {} of {})'.format(results_current_page + 1, results_num_pages + 1))
|
||||
print(
|
||||
"Indexing item downloads (page {} of {})".format(
|
||||
results_current_page + 1, results_num_pages + 1
|
||||
)
|
||||
)
|
||||
|
||||
solr_query_params = {
|
||||
'q': 'type:0',
|
||||
'fq': 'isBot:false AND statistics_type:view AND bundleName:ORIGINAL',
|
||||
'facet': 'true',
|
||||
'facet.field': 'owningItem',
|
||||
'facet.mincount': 1,
|
||||
'facet.limit': results_per_page,
|
||||
'facet.offset': results_current_page * results_per_page,
|
||||
'shards': shards,
|
||||
'rows': 0,
|
||||
'wt': 'json',
|
||||
'json.nl': 'map' # return facets as a dict instead of a flat list
|
||||
"q": "type:0",
|
||||
"fq": "isBot:false AND statistics_type:view AND bundleName:ORIGINAL",
|
||||
"facet": "true",
|
||||
"facet.field": "owningItem",
|
||||
"facet.mincount": 1,
|
||||
"facet.limit": results_per_page,
|
||||
"facet.offset": results_current_page * results_per_page,
|
||||
"shards": shards,
|
||||
"rows": 0,
|
||||
"wt": "json",
|
||||
"json.nl": "map", # return facets as a dict instead of a flat list
|
||||
}
|
||||
|
||||
solr_url = SOLR_SERVER + '/statistics/select'
|
||||
solr_url = SOLR_SERVER + "/statistics/select"
|
||||
|
||||
res = requests.get(solr_url, params=solr_query_params)
|
||||
|
||||
# Solr returns facets as a dict of dicts (see json.nl parameter)
|
||||
downloads = res.json()['facet_counts']['facet_fields']
|
||||
downloads = res.json()["facet_counts"]["facet_fields"]
|
||||
# iterate over the 'owningItem' dict and get the item ids and downloads
|
||||
for item_id, item_downloads in downloads['owningItem'].items():
|
||||
for item_id, item_downloads in downloads["owningItem"].items():
|
||||
data.append((item_id, item_downloads))
|
||||
|
||||
# do a batch insert of values from the current "page" of results
|
||||
sql = 'INSERT INTO items(id, downloads) VALUES %s ON CONFLICT(id) DO UPDATE SET downloads=excluded.downloads'
|
||||
psycopg2.extras.execute_values(cursor, sql, data, template='(%s, %s)')
|
||||
sql = "INSERT INTO items(id, downloads) VALUES %s ON CONFLICT(id) DO UPDATE SET downloads=excluded.downloads"
|
||||
psycopg2.extras.execute_values(cursor, sql, data, template="(%s, %s)")
|
||||
db.commit()
|
||||
|
||||
# clear all items from the list so we can populate it with the next batch
|
||||
@ -252,8 +262,10 @@ def index_downloads():
|
||||
with DatabaseManager() as db:
|
||||
with db.cursor() as cursor:
|
||||
# create table to store item views and downloads
|
||||
cursor.execute('''CREATE TABLE IF NOT EXISTS items
|
||||
(id INT PRIMARY KEY, views INT DEFAULT 0, downloads INT DEFAULT 0)''')
|
||||
cursor.execute(
|
||||
"""CREATE TABLE IF NOT EXISTS items
|
||||
(id INT PRIMARY KEY, views INT DEFAULT 0, downloads INT DEFAULT 0)"""
|
||||
)
|
||||
|
||||
# commit the table creation before closing the database connection
|
||||
db.commit()
|
||||
|
@ -1,26 +1,27 @@
|
||||
-i https://pypi.org/simple
|
||||
atomicwrites==1.3.0
|
||||
attrs==19.1.0
|
||||
attrs==19.3.0
|
||||
backcall==0.1.0
|
||||
decorator==4.4.0
|
||||
decorator==4.4.1
|
||||
entrypoints==0.3
|
||||
flake8==3.7.7
|
||||
flake8==3.7.9
|
||||
ipython-genutils==0.2.0
|
||||
ipython==7.4.0
|
||||
jedi==0.13.3
|
||||
ipython==7.9.0
|
||||
jedi==0.15.1
|
||||
mccabe==0.6.1
|
||||
more-itertools==7.0.0 ; python_version > '2.7'
|
||||
parso==0.4.0
|
||||
more-itertools==7.2.0
|
||||
packaging==19.2
|
||||
parso==0.5.1
|
||||
pexpect==4.7.0 ; sys_platform != 'win32'
|
||||
pickleshare==0.7.5
|
||||
pluggy==0.9.0
|
||||
prompt-toolkit==2.0.9
|
||||
pluggy==0.13.1
|
||||
prompt-toolkit==2.0.10
|
||||
ptyprocess==0.6.0
|
||||
py==1.8.0
|
||||
pycodestyle==2.5.0
|
||||
pyflakes==2.1.1
|
||||
pygments==2.3.1
|
||||
pytest==4.4.0
|
||||
six==1.12.0
|
||||
traitlets==4.3.2
|
||||
pygments==2.5.1
|
||||
pyparsing==2.4.5
|
||||
pytest==5.3.1
|
||||
six==1.13.0
|
||||
traitlets==4.3.3
|
||||
wcwidth==0.1.7
|
||||
|
@ -1,11 +1,9 @@
|
||||
-i https://pypi.org/simple
|
||||
certifi==2019.3.9
|
||||
certifi==2019.9.11
|
||||
chardet==3.0.4
|
||||
falcon==1.4.1
|
||||
gunicorn==19.9.0
|
||||
falcon==2.0.0
|
||||
gunicorn==20.0.4
|
||||
idna==2.8
|
||||
psycopg2-binary==2.8.2
|
||||
python-mimeparse==1.6.0
|
||||
requests==2.21.0
|
||||
six==1.12.0
|
||||
urllib3==1.24.1
|
||||
psycopg2-binary==2.8.4
|
||||
requests==2.22.0
|
||||
urllib3==1.25.7
|
||||
|
Reference in New Issue
Block a user