From 519d9c84a95375b275b2c86ddc795aeba4a08ca6 Mon Sep 17 00:00:00 2001
From: Alan Orth
Date: Thu, 14 Nov 2019 09:47:49 +0200
Subject: [PATCH] Update notes for 2019-11-14
---
content/posts/2019-11.md | 14 ++++++++++++++
docs/2019-11/index.html | 26 +++++++++++++++++++++-----
docs/sitemap.xml | 10 +++++-----
3 files changed, 40 insertions(+), 10 deletions(-)
diff --git a/content/posts/2019-11.md b/content/posts/2019-11.md
index 0af69d2c0..6640a9c8a 100644
--- a/content/posts/2019-11.md
+++ b/content/posts/2019-11.md
@@ -288,5 +288,19 @@ $ http "http://localhost:8081/solr/statistics/select?q=userAgent:/Scrapoo\/[0-9]
```
- Nice, so searching with regex in Solr with `//` syntax works for those digits!
+- I realized that it's easier to search Solr from curl via POST using this syntax:
+
+```
+$ curl -s "http://localhost:8081/solr/statistics/select" -d "q=userAgent:*Scrapoo*&rows=0")
+```
+
+- If the parameters include something like "[0-9]" then curl interprets it as a range and will make ten requests
+ - You can disable this using the `-g` option, but there are other benefits to searching with POST, for example it seems that I have less issues with escaping special parameters when using Solr's regex search:
+
+```
+$ curl -s 'http://localhost:8081/solr/statistics/select' -d 'q=userAgent:/Postgenomic(\s|\+)v2/&rows=2'
+```
+
+- I updated the `check-spider-hits.sh` script to use the POST syntax, and I'm evaluating the feasability of including the regex search patterns from the spider agent file, as I had been filtering them out due to differences in PCRE and Solr regex syntax and issues with shell handling
diff --git a/docs/2019-11/index.html b/docs/2019-11/index.html
index be9c801b9..ec6dfded6 100644
--- a/docs/2019-11/index.html
+++ b/docs/2019-11/index.html
@@ -34,7 +34,7 @@ Let’s see how many of the REST API requests were for bitstreams (because t
-
+
@@ -73,9 +73,9 @@ Let’s see how many of the REST API requests were for bitstreams (because t
"@type": "BlogPosting",
"headline": "November, 2019",
"url": "https:\/\/alanorth.github.io\/cgspace-notes\/2019-11\/",
- "wordCount": "1819",
+ "wordCount": "1951",
"datePublished": "2019-11-04T12:20:30+02:00",
- "dateModified": "2019-11-12T11:44:05+02:00",
+ "dateModified": "2019-11-13T18:18:24+02:00",
"author": {
"@type": "Person",
"name": "Alan Orth"
@@ -446,11 +446,27 @@ $ http “http://localhost:8081/solr/statistics/select?q=userAgent:Scrapoo*" | xmllint –format - | grep numFound
$ http “http://localhost:8081/solr/statistics/select?q=userAgent:/Scrapoo/[0-9]/" | xmllint –format - | grep numFound
-
+
+
+
+- Nice, so searching with regex in Solr with `//` syntax works for those digits!
+- I realized that it's easier to search Solr from curl via POST using this syntax:
+
+
+
+$ curl -s “http://localhost:8081/solr/statistics/select" -d “q=userAgent:Scrapoo&rows=0”)
+
+
+- If the parameters include something like "[0-9]" then curl interprets it as a range and will make ten requests
+ - You can disable this using the `-g` option, but there are other benefits to searching with POST, for example it seems that I have less issues with escaping special parameters when using Solr's regex search:
+
+
+
+$ curl -s ‘http://localhost:8081/solr/statistics/select' -d ‘q=userAgent:/Postgenomic(\s|+)v2/&rows=2’
```
-- Nice, so searching with regex in Solr with
//
syntax works for those digits!
+- I updated the
check-spider-hits.sh
script to use the POST syntax, and I’m evaluating the feasability of including the regex search patterns from the spider agent file, as I had been filtering them out due to differences in PCRE and Solr regex syntax and issues with shell handling
diff --git a/docs/sitemap.xml b/docs/sitemap.xml
index a63ac8458..f39beb64c 100644
--- a/docs/sitemap.xml
+++ b/docs/sitemap.xml
@@ -4,27 +4,27 @@
https://alanorth.github.io/cgspace-notes/categories/
- 2019-11-12T11:44:05+02:00
+ 2019-11-13T18:18:24+02:00
https://alanorth.github.io/cgspace-notes/
- 2019-11-12T11:44:05+02:00
+ 2019-11-13T18:18:24+02:00
https://alanorth.github.io/cgspace-notes/categories/notes/
- 2019-11-12T11:44:05+02:00
+ 2019-11-13T18:18:24+02:00
https://alanorth.github.io/cgspace-notes/2019-11/
- 2019-11-12T11:44:05+02:00
+ 2019-11-13T18:18:24+02:00
https://alanorth.github.io/cgspace-notes/posts/
- 2019-11-12T11:44:05+02:00
+ 2019-11-13T18:18:24+02:00