From 3d966079204ad2d673e7ab21839ca715f10d566d Mon Sep 17 00:00:00 2001 From: Alan Orth Date: Sun, 5 Jul 2020 10:50:09 +0300 Subject: [PATCH] Update notes for 2020-07-03 --- content/posts/2020-07.md | 40 ++++++++++++++++++++++ docs/2020-07/index.html | 44 ++++++++++++++++++++++--- docs/categories/index.html | 2 +- docs/categories/notes/index.html | 2 +- docs/categories/notes/page/2/index.html | 2 +- docs/categories/notes/page/3/index.html | 2 +- docs/categories/notes/page/4/index.html | 2 +- docs/index.html | 2 +- docs/page/2/index.html | 2 +- docs/page/3/index.html | 2 +- docs/page/4/index.html | 2 +- docs/page/5/index.html | 2 +- docs/page/6/index.html | 2 +- docs/posts/index.html | 2 +- docs/posts/page/2/index.html | 2 +- docs/posts/page/3/index.html | 2 +- docs/posts/page/4/index.html | 2 +- docs/posts/page/5/index.html | 2 +- docs/posts/page/6/index.html | 2 +- docs/sitemap.xml | 10 +++--- 20 files changed, 102 insertions(+), 26 deletions(-) diff --git a/content/posts/2020-07.md b/content/posts/2020-07.md index 6e9bd788b..33e9d377a 100644 --- a/content/posts/2020-07.md +++ b/content/posts/2020-07.md @@ -178,5 +178,45 @@ Typhoeus - Atmire says they are able to build fine, so I tried again and noticed that I had been building with `-Denv=dspacetest.cgiar.org`, which is not necessary for DSpace 6 of course - Once I removed that it builds fine - I quickly re-applied the Font Awesome 5 changes to use SVG+JS instead of web fonts (from 2020-04) and things are looking good! +- Run all system updates on DSpace Test (linode26), deploy latest `6_x-dev-atmire-modules` branch, and reboot it + +## 2020-07-02 + +- I need to export some Solr statistics data from CGSpace to test Salem's modifications to the dspace-statistics-api + - He modified it to query Solr on the fly instead of indexing it, which will be heavier and slower, but allows us to get more granular stats and countries/cities + - Because have so many records I want to use solr-import-export-json to get several months at a time with a date range, but it seems there are first issues with curl (need to disable globbing with `-g` and URL encode the range) + - For reference, the [Solr 4.10.x DateField docs](https://lucene.apache.org/solr/4_10_2/solr-core/org/apache/solr/schema/DateField.html) + - This range works in Solr UI: `[2019-01-01T00:00:00Z TO 2019-06-30T23:59:59Z]` + - As well in curl: + +``` +$ curl -g -s 'http://localhost:8081/solr/statistics-2019/select?q=*:*&fq=time:%5B2019-01-01T00%3A00%3A00Z%20TO%202019-06-30T23%3A59%3A59Z%5D&rows=0&wt=json&indent=true' +{ + "responseHeader":{ + "status":0, + "QTime":0, + "params":{ + "q":"*:*", + "indent":"true", + "fq":"time:[2019-01-01T00:00:00Z TO 2019-06-30T23:59:59Z]", + "rows":"0", + "wt":"json"}}, + "response":{"numFound":7784285,"start":0,"docs":[] + }} +``` + +- But not in solr-import-export-json... hmmm... seems we need to URL encode *only* the date range itself, but not the brackets: + +``` +$ ./run.sh -s http://localhost:8081/solr/statistics-2019 -a export -o /tmp/statistics-2019-1.json -f 'time:%5B2019-01-01T00%3A00%3A00Z%20TO%202019-06-30T23%3A59%3A59Z]' -k uid +$ zstd /tmp/statistics-2019-1.json +``` + +- Then import it on my local dev environment: + +``` +$ zstd -d statistics-2019-1.json.zst +$ ./run.sh -s http://localhost:8080/solr/statistics -a import -o ~/Downloads/statistics-2019-1.json -k uid +``` diff --git a/docs/2020-07/index.html b/docs/2020-07/index.html index 9b1d60d99..45545c8a8 100644 --- a/docs/2020-07/index.html +++ b/docs/2020-07/index.html @@ -20,7 +20,7 @@ Since I was restarting Tomcat anyways I decided to redeploy the latest changes f - + @@ -45,9 +45,9 @@ Since I was restarting Tomcat anyways I decided to redeploy the latest changes f "@type": "BlogPosting", "headline": "July, 2020", "url": "https://alanorth.github.io/cgspace-notes/2020-07/", - "wordCount": "844", + "wordCount": "1043", "datePublished": "2020-07-01T10:53:54+03:00", - "dateModified": "2020-07-01T15:37:20+03:00", + "dateModified": "2020-07-02T09:53:45+03:00", "author": { "@type": "Person", "name": "Alan Orth" @@ -291,8 +291,44 @@ Typhoeus
  • I quickly re-applied the Font Awesome 5 changes to use SVG+JS instead of web fonts (from 2020-04) and things are looking good!
    • +
  • Run all system updates on DSpace Test (linode26), deploy latest 6_x-dev-atmire-modules branch, and reboot it
    • - +

    2020-07-02

    + +
    $ curl -g -s 'http://localhost:8081/solr/statistics-2019/select?q=*:*&fq=time:%5B2019-01-01T00%3A00%3A00Z%20TO%202019-06-30T23%3A59%3A59Z%5D&rows=0&wt=json&indent=true'
+{
+  "responseHeader":{
+    "status":0,
+    "QTime":0,
+    "params":{
+      "q":"*:*",
+      "indent":"true",
+      "fq":"time:[2019-01-01T00:00:00Z TO 2019-06-30T23:59:59Z]",
+      "rows":"0",
+      "wt":"json"}},
+  "response":{"numFound":7784285,"start":0,"docs":[]
+  }}
+
    +
    $ ./run.sh -s http://localhost:8081/solr/statistics-2019 -a export -o /tmp/statistics-2019-1.json -f 'time:%5B2019-01-01T00%3A00%3A00Z%20TO%202019-06-30T23%3A59%3A59Z]' -k uid
+$ zstd /tmp/statistics-2019-1.json
+
    +
    $ zstd -d statistics-2019-1.json.zst
+$ ./run.sh -s http://localhost:8080/solr/statistics -a import -o ~/Downloads/statistics-2019-1.json -k uid
+
    diff --git a/docs/categories/index.html b/docs/categories/index.html index d7fb9e7de..4718d0298 100644 --- a/docs/categories/index.html +++ b/docs/categories/index.html @@ -9,7 +9,7 @@ - + diff --git a/docs/categories/notes/index.html b/docs/categories/notes/index.html index a2148d847..861c146a4 100644 --- a/docs/categories/notes/index.html +++ b/docs/categories/notes/index.html @@ -9,7 +9,7 @@ - + diff --git a/docs/categories/notes/page/2/index.html b/docs/categories/notes/page/2/index.html index 0cdc5727f..5db09e3e7 100644 --- a/docs/categories/notes/page/2/index.html +++ b/docs/categories/notes/page/2/index.html @@ -9,7 +9,7 @@ - + diff --git a/docs/categories/notes/page/3/index.html b/docs/categories/notes/page/3/index.html index 3dd785c78..afd629151 100644 --- a/docs/categories/notes/page/3/index.html +++ b/docs/categories/notes/page/3/index.html @@ -9,7 +9,7 @@ - + diff --git a/docs/categories/notes/page/4/index.html b/docs/categories/notes/page/4/index.html index ed5dd9284..1ff51521a 100644 --- a/docs/categories/notes/page/4/index.html +++ b/docs/categories/notes/page/4/index.html @@ -9,7 +9,7 @@ - + diff --git a/docs/index.html b/docs/index.html index 64ba0aebf..79272cd46 100644 --- a/docs/index.html +++ b/docs/index.html @@ -9,7 +9,7 @@ - + diff --git a/docs/page/2/index.html b/docs/page/2/index.html index dd50e5d45..8500a156e 100644 --- a/docs/page/2/index.html +++ b/docs/page/2/index.html @@ -9,7 +9,7 @@ - + diff --git a/docs/page/3/index.html b/docs/page/3/index.html index fcf28e337..32a9a6893 100644 --- a/docs/page/3/index.html +++ b/docs/page/3/index.html @@ -9,7 +9,7 @@ - + diff --git a/docs/page/4/index.html b/docs/page/4/index.html index 2c874106b..2fa345f54 100644 --- a/docs/page/4/index.html +++ b/docs/page/4/index.html @@ -9,7 +9,7 @@ - + diff --git a/docs/page/5/index.html b/docs/page/5/index.html index 096f2594e..e05d51693 100644 --- a/docs/page/5/index.html +++ b/docs/page/5/index.html @@ -9,7 +9,7 @@ - + diff --git a/docs/page/6/index.html b/docs/page/6/index.html index ec40d155c..e4d7bbc06 100644 --- a/docs/page/6/index.html +++ b/docs/page/6/index.html @@ -9,7 +9,7 @@ - + diff --git a/docs/posts/index.html b/docs/posts/index.html index cea31fe89..7015d4ea3 100644 --- a/docs/posts/index.html +++ b/docs/posts/index.html @@ -9,7 +9,7 @@ - + diff --git a/docs/posts/page/2/index.html b/docs/posts/page/2/index.html index 534e2c058..6e339a4ce 100644 --- a/docs/posts/page/2/index.html +++ b/docs/posts/page/2/index.html @@ -9,7 +9,7 @@ - + diff --git a/docs/posts/page/3/index.html b/docs/posts/page/3/index.html index 2308050cc..19a134812 100644 --- a/docs/posts/page/3/index.html +++ b/docs/posts/page/3/index.html @@ -9,7 +9,7 @@ - + diff --git a/docs/posts/page/4/index.html b/docs/posts/page/4/index.html index 821350e64..62d5564d0 100644 --- a/docs/posts/page/4/index.html +++ b/docs/posts/page/4/index.html @@ -9,7 +9,7 @@ - + diff --git a/docs/posts/page/5/index.html b/docs/posts/page/5/index.html index 08c787b15..c5afaeb4f 100644 --- a/docs/posts/page/5/index.html +++ b/docs/posts/page/5/index.html @@ -9,7 +9,7 @@ - + diff --git a/docs/posts/page/6/index.html b/docs/posts/page/6/index.html index d88405ba2..924c8754c 100644 --- a/docs/posts/page/6/index.html +++ b/docs/posts/page/6/index.html @@ -9,7 +9,7 @@ - + diff --git a/docs/sitemap.xml b/docs/sitemap.xml index efce3e17b..a33c814f2 100644 --- a/docs/sitemap.xml +++ b/docs/sitemap.xml @@ -4,27 +4,27 @@ https://alanorth.github.io/cgspace-notes/categories/ - 2020-07-01T15:37:20+03:00 + 2020-07-02T09:53:45+03:00 https://alanorth.github.io/cgspace-notes/ - 2020-07-01T15:37:20+03:00 + 2020-07-02T09:53:45+03:00 https://alanorth.github.io/cgspace-notes/2020-07/ - 2020-07-01T15:37:20+03:00 + 2020-07-02T09:53:45+03:00 https://alanorth.github.io/cgspace-notes/categories/notes/ - 2020-07-01T15:37:20+03:00 + 2020-07-02T09:53:45+03:00 https://alanorth.github.io/cgspace-notes/posts/ - 2020-07-01T15:37:20+03:00 + 2020-07-02T09:53:45+03:00