From 7cdb9f31e696b24d0bd18a23740c7102395ebc75 Mon Sep 17 00:00:00 2001 From: Alan Orth Date: Mon, 19 Oct 2020 17:22:49 +0300 Subject: [PATCH] Update notes for 2020-10-19 --- content/posts/2020-10.md | 20 ++++++++++++++ docs/2020-10/index.html | 36 ++++++++++++++++++++++--- docs/categories/index.html | 2 +- docs/categories/notes/index.html | 2 +- docs/categories/notes/page/2/index.html | 2 +- docs/categories/notes/page/3/index.html | 2 +- docs/categories/notes/page/4/index.html | 2 +- docs/index.html | 2 +- docs/page/2/index.html | 2 +- docs/page/3/index.html | 2 +- docs/page/4/index.html | 2 +- docs/page/5/index.html | 2 +- docs/page/6/index.html | 2 +- docs/page/7/index.html | 2 +- docs/posts/index.html | 2 +- docs/posts/page/2/index.html | 2 +- docs/posts/page/3/index.html | 2 +- docs/posts/page/4/index.html | 2 +- docs/posts/page/5/index.html | 2 +- docs/posts/page/6/index.html | 2 +- docs/posts/page/7/index.html | 2 +- docs/sitemap.xml | 10 +++---- 22 files changed, 76 insertions(+), 28 deletions(-) diff --git a/content/posts/2020-10.md b/content/posts/2020-10.md index b26c69f8a..c6f8087e6 100644 --- a/content/posts/2020-10.md +++ b/content/posts/2020-10.md @@ -589,4 +589,24 @@ Purging 1282 hits from curl in statistics Total number of bot hits purged: 8174 ``` +- Add "Infographic" to types in input form +- Looking into the spider agent issue from last week, where hits seem to be logged regardless of ANY spider agent patterns being loaded + - I changed the following two options: + - `usage-statistics.logBots = false` + - `usage-statistics.bots.case-insensitive = true` + - Then I made several requests with a bot user agent: + +``` +$ http --print Hh https://dspacetest.cgiar.org/rest/bitstreams/dfa1d9c3-75d3-4380-a9d3-4c8cbbed2d21/retrieve User-Agent:"RTB website BOT" +$ curl -s 'http://localhost:8083/solr/statistics/update?softCommit=true' +``` + +- And I saw three hits in Solr with `isBot: true`!!! + - I made a few more requests with user agent "fumanchu" and it logs them with `isBot: false`... + - I made a request with user agent "Delphi 2009" which is in the ilri pattern file, and it was logged with `isBot: true` + - I made a few more requests and confirmed that if a pattern is in the list it gets logged with `isBot: true` despite the fact that `usage-statistics.logBots` is false... + - So WTF this means that it *knows* they are from a bot, but it logs them anyways + - Is this an issue with Atmire's modules? + - I sent them feedback on the ticket + diff --git a/docs/2020-10/index.html b/docs/2020-10/index.html index 09b386f21..203f9ac12 100644 --- a/docs/2020-10/index.html +++ b/docs/2020-10/index.html @@ -23,7 +23,7 @@ During the FlywayDB migration I got an error: - + @@ -51,9 +51,9 @@ During the FlywayDB migration I got an error: "@type": "BlogPosting", "headline": "October, 2020", "url": "https://alanorth.github.io/cgspace-notes/2020-10/", - "wordCount": "3789", + "wordCount": "3963", "datePublished": "2020-10-06T16:55:54+03:00", - "dateModified": "2020-10-15T18:11:00+03:00", + "dateModified": "2020-10-19T15:47:59+03:00", "author": { "@type": "Person", "name": "Alan Orth" @@ -776,7 +776,35 @@ Purging 1851 hits from ILRI Livestock Website Publications importer BOT in stati Purging 1282 hits from curl in statistics Total number of bot hits purged: 8174 - + +
$ http --print Hh https://dspacetest.cgiar.org/rest/bitstreams/dfa1d9c3-75d3-4380-a9d3-4c8cbbed2d21/retrieve User-Agent:"RTB website BOT"
+$ curl -s 'http://localhost:8083/solr/statistics/update?softCommit=true'
+
+ diff --git a/docs/categories/index.html b/docs/categories/index.html index ee7c9d038..e2f98f93d 100644 --- a/docs/categories/index.html +++ b/docs/categories/index.html @@ -9,7 +9,7 @@ - + diff --git a/docs/categories/notes/index.html b/docs/categories/notes/index.html index 680d83ec3..a86093597 100644 --- a/docs/categories/notes/index.html +++ b/docs/categories/notes/index.html @@ -9,7 +9,7 @@ - + diff --git a/docs/categories/notes/page/2/index.html b/docs/categories/notes/page/2/index.html index f2d4d8e6d..a8ed41f3a 100644 --- a/docs/categories/notes/page/2/index.html +++ b/docs/categories/notes/page/2/index.html @@ -9,7 +9,7 @@ - + diff --git a/docs/categories/notes/page/3/index.html b/docs/categories/notes/page/3/index.html index 45d2722d4..1557aa375 100644 --- a/docs/categories/notes/page/3/index.html +++ b/docs/categories/notes/page/3/index.html @@ -9,7 +9,7 @@ - + diff --git a/docs/categories/notes/page/4/index.html b/docs/categories/notes/page/4/index.html index 11ec1e73d..f56c170e6 100644 --- a/docs/categories/notes/page/4/index.html +++ b/docs/categories/notes/page/4/index.html @@ -9,7 +9,7 @@ - + diff --git a/docs/index.html b/docs/index.html index 2aa959819..c4892de8c 100644 --- a/docs/index.html +++ b/docs/index.html @@ -9,7 +9,7 @@ - + diff --git a/docs/page/2/index.html b/docs/page/2/index.html index 67c12a019..a632067f7 100644 --- a/docs/page/2/index.html +++ b/docs/page/2/index.html @@ -9,7 +9,7 @@ - + diff --git a/docs/page/3/index.html b/docs/page/3/index.html index 38610caee..9c61141c1 100644 --- a/docs/page/3/index.html +++ b/docs/page/3/index.html @@ -9,7 +9,7 @@ - + diff --git a/docs/page/4/index.html b/docs/page/4/index.html index fe3e8fb25..9d74f825d 100644 --- a/docs/page/4/index.html +++ b/docs/page/4/index.html @@ -9,7 +9,7 @@ - + diff --git a/docs/page/5/index.html b/docs/page/5/index.html index cfa7b1774..dd4b08a51 100644 --- a/docs/page/5/index.html +++ b/docs/page/5/index.html @@ -9,7 +9,7 @@ - + diff --git a/docs/page/6/index.html b/docs/page/6/index.html index 1d3a91740..11e8124e7 100644 --- a/docs/page/6/index.html +++ b/docs/page/6/index.html @@ -9,7 +9,7 @@ - + diff --git a/docs/page/7/index.html b/docs/page/7/index.html index 0c3ff519d..0cfc235e5 100644 --- a/docs/page/7/index.html +++ b/docs/page/7/index.html @@ -9,7 +9,7 @@ - + diff --git a/docs/posts/index.html b/docs/posts/index.html index d0fdd6188..14a4f9ad2 100644 --- a/docs/posts/index.html +++ b/docs/posts/index.html @@ -9,7 +9,7 @@ - + diff --git a/docs/posts/page/2/index.html b/docs/posts/page/2/index.html index cab1ada97..b36180d28 100644 --- a/docs/posts/page/2/index.html +++ b/docs/posts/page/2/index.html @@ -9,7 +9,7 @@ - + diff --git a/docs/posts/page/3/index.html b/docs/posts/page/3/index.html index 865cee404..e1984de5a 100644 --- a/docs/posts/page/3/index.html +++ b/docs/posts/page/3/index.html @@ -9,7 +9,7 @@ - + diff --git a/docs/posts/page/4/index.html b/docs/posts/page/4/index.html index 62aff9f44..44afe66c0 100644 --- a/docs/posts/page/4/index.html +++ b/docs/posts/page/4/index.html @@ -9,7 +9,7 @@ - + diff --git a/docs/posts/page/5/index.html b/docs/posts/page/5/index.html index c6e100bc9..89ad54bc4 100644 --- a/docs/posts/page/5/index.html +++ b/docs/posts/page/5/index.html @@ -9,7 +9,7 @@ - + diff --git a/docs/posts/page/6/index.html b/docs/posts/page/6/index.html index 917f1ef1b..14df487d7 100644 --- a/docs/posts/page/6/index.html +++ b/docs/posts/page/6/index.html @@ -9,7 +9,7 @@ - + diff --git a/docs/posts/page/7/index.html b/docs/posts/page/7/index.html index fee7f1f3e..f1e72f571 100644 --- a/docs/posts/page/7/index.html +++ b/docs/posts/page/7/index.html @@ -9,7 +9,7 @@ - + diff --git a/docs/sitemap.xml b/docs/sitemap.xml index a7b4bebdd..4dd39fc51 100644 --- a/docs/sitemap.xml +++ b/docs/sitemap.xml @@ -4,27 +4,27 @@ https://alanorth.github.io/cgspace-notes/categories/ - 2020-10-19T15:23:30+03:00 + 2020-10-19T15:47:59+03:00 https://alanorth.github.io/cgspace-notes/ - 2020-10-19T15:23:30+03:00 + 2020-10-19T15:47:59+03:00 https://alanorth.github.io/cgspace-notes/categories/notes/ - 2020-10-19T15:23:30+03:00 + 2020-10-19T15:47:59+03:00 https://alanorth.github.io/cgspace-notes/2020-10/ - 2020-10-15T18:11:00+03:00 + 2020-10-19T15:47:59+03:00 https://alanorth.github.io/cgspace-notes/posts/ - 2020-10-19T15:23:30+03:00 + 2020-10-19T15:47:59+03:00