From 784984f4c041a874f141e7316cc867c4b471feb0 Mon Sep 17 00:00:00 2001 From: Alan Orth Date: Sun, 24 Oct 2021 21:21:01 +0300 Subject: [PATCH] Add notes for 2021-10-24 --- content/posts/2021-10.md | 43 ++++++++++++++++++ docs/2021-10/index.html | 60 +++++++++++++++++++++++-- docs/categories/index.html | 2 +- docs/categories/notes/index.html | 2 +- docs/categories/notes/page/2/index.html | 2 +- docs/categories/notes/page/3/index.html | 2 +- docs/categories/notes/page/4/index.html | 2 +- docs/categories/notes/page/5/index.html | 2 +- docs/categories/notes/page/6/index.html | 2 +- docs/index.html | 2 +- docs/page/2/index.html | 2 +- docs/page/3/index.html | 2 +- docs/page/4/index.html | 2 +- docs/page/5/index.html | 2 +- docs/page/6/index.html | 2 +- docs/page/7/index.html | 2 +- docs/page/8/index.html | 2 +- docs/posts/index.html | 2 +- docs/posts/page/2/index.html | 2 +- docs/posts/page/3/index.html | 2 +- docs/posts/page/4/index.html | 2 +- docs/posts/page/5/index.html | 2 +- docs/posts/page/6/index.html | 2 +- docs/posts/page/7/index.html | 2 +- docs/posts/page/8/index.html | 2 +- docs/sitemap.xml | 10 ++--- 26 files changed, 128 insertions(+), 31 deletions(-) diff --git a/content/posts/2021-10.md b/content/posts/2021-10.md index dfc9808a1..5456f43c4 100644 --- a/content/posts/2021-10.md +++ b/content/posts/2021-10.md @@ -479,4 +479,47 @@ Time: 4417.909 ms (00:04.418) - We discussed several options, for example moving all DSpaces to CGSpace along with their permanent identifiers - The issue would be for centers like IFPRI who don't use DSpace and have integrations with their website etc with their current repository +## 2021-10-21 + +- Udana from IWMI contacted me to ask if I could do a one-off AReS harvest because they have some new items they need to report on + +## 2021-10-22 + +- Abenet and others contacted me to say that the LDAP login was not working on CGSpace + - I checked with `ldapsearch` and it is indeed not working: + +```console +$ ldapsearch -x -H ldaps://AZCGNEROOT3.CGIARAD.ORG:636/ -b "dc=cgiarad,dc=org" -D "booo" -W "(sAMAccountName=fuuu)" +Enter LDAP Password: +ldap_bind: Invalid credentials (49) + additional info: 80090308: LdapErr: DSID-0C090447, comment: AcceptSecurityContext error, data 52e, v3839 +``` + +- I sent a message to ILRI ICT to ask them to check the account + - They reset the password so I ran all system updates and rebooted the server since users weren't able to log in anyways + +## 2021-10-24 + +- CIP was asking about CGSpace stats again + - The last time I helped them with this was in 2021-04, when I extracted stats for their community from the DSpace Statistics API +- In looking at the CIP stats request I got curious if there were any hits from all those Russian IPs before 2021-07 that I could purge + - Sure enough there were a few hundred IPs belonging to those ASNs: + +```console +$ http 'localhost:8081/solr/statistics/select?q=time%3A2021-04*&fl=ip&wt=json&indent=true&facet=true&facet.field=ip&facet.limit=200000&facet.mincount=1' > /tmp/2021-04-ips.json +# Ghetto way to extract the IPs using jq, but I can't figure out how only print them and not the facet counts, so I just use sed +$ jq '.facet_counts.facet_fields.ip[]' /tmp/2021-04-ips.json | grep -E '^"' | sed -e 's/"//g' > /tmp/ips.txt +$ ./ilri/resolve-addresses-geoip2.py -i /tmp/ips.txt -o /tmp/2021-04-ips.csv +$ csvgrep -c asn -r '^(49453|46844|206485|62282|36352|35913|35624|8100)$' /tmp/2021-04-ips.csv | csvcut -c network | sed 1d | sort -u > /tmp/networks-to-block.txt +$ wc -l /tmp/networks-to-block.txt +125 /tmp/networks-to-block.txt +$ grepcidr -f /tmp/networks-to-block.txt /tmp/ips.txt > /tmp/ips-to-purge.txt +$ wc -l /tmp/ips-to-purge.txt +202 +``` + +- Attempting to purge those only shows about 3,500 hits, but I will do it anyways + - Adding 64.39.108.48 from Qualys I get a total of 22631 hits purged +- I also purged another 5306 hits after checking the IPv4 list from AbuseIPDB.com + diff --git a/docs/2021-10/index.html b/docs/2021-10/index.html index c05fe9e95..f13c3a3b9 100644 --- a/docs/2021-10/index.html +++ b/docs/2021-10/index.html @@ -25,7 +25,7 @@ So we have 1879/7100 (26.46%) matching already - + @@ -56,9 +56,9 @@ So we have 1879/7100 (26.46%) matching already "@type": "BlogPosting", "headline": "October, 2021", "url": "https://alanorth.github.io/cgspace-notes/2021-10/", - "wordCount": "3392", + "wordCount": "3710", "datePublished": "2021-10-01T11:14:07+03:00", - "dateModified": "2021-10-17T20:47:01+03:00", + "dateModified": "2021-10-20T22:21:55+03:00", "author": { "@type": "Person", "name": "Alan Orth" @@ -628,6 +628,60 @@ Time: 4417.909 ms (00:04.418) +

2021-10-21

+ +

2021-10-22

+ +
$ ldapsearch -x -H ldaps://AZCGNEROOT3.CGIARAD.ORG:636/ -b "dc=cgiarad,dc=org" -D "booo" -W "(sAMAccountName=fuuu)"
+Enter LDAP Password:
+ldap_bind: Invalid credentials (49)
+        additional info: 80090308: LdapErr: DSID-0C090447, comment: AcceptSecurityContext error, data 52e, v3839
+
+

2021-10-24

+ +
$ http 'localhost:8081/solr/statistics/select?q=time%3A2021-04*&fl=ip&wt=json&indent=true&facet=true&facet.field=ip&facet.limit=200000&facet.mincount=1' > /tmp/2021-04-ips.json
+# Ghetto way to extract the IPs using jq, but I can't figure out how only print them and not the facet counts, so I just use sed
+$ jq '.facet_counts.facet_fields.ip[]' /tmp/2021-04-ips.json | grep -E '^"' | sed -e 's/"//g' > /tmp/ips.txt
+$ ./ilri/resolve-addresses-geoip2.py -i /tmp/ips.txt -o /tmp/2021-04-ips.csv
+$ csvgrep -c asn -r '^(49453|46844|206485|62282|36352|35913|35624|8100)$' /tmp/2021-04-ips.csv | csvcut -c network | sed 1d | sort -u > /tmp/networks-to-block.txt
+$ wc -l /tmp/networks-to-block.txt 
+125 /tmp/networks-to-block.txt
+$ grepcidr -f /tmp/networks-to-block.txt /tmp/ips.txt > /tmp/ips-to-purge.txt
+$ wc -l /tmp/ips-to-purge.txt
+202
+
diff --git a/docs/categories/index.html b/docs/categories/index.html index efe59395c..be81a4f86 100644 --- a/docs/categories/index.html +++ b/docs/categories/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/categories/notes/index.html b/docs/categories/notes/index.html index 4f742ad78..ed0963c62 100644 --- a/docs/categories/notes/index.html +++ b/docs/categories/notes/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/categories/notes/page/2/index.html b/docs/categories/notes/page/2/index.html index 72fec8193..ce59d4f2d 100644 --- a/docs/categories/notes/page/2/index.html +++ b/docs/categories/notes/page/2/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/categories/notes/page/3/index.html b/docs/categories/notes/page/3/index.html index 4aa69a021..da08ea5ce 100644 --- a/docs/categories/notes/page/3/index.html +++ b/docs/categories/notes/page/3/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/categories/notes/page/4/index.html b/docs/categories/notes/page/4/index.html index 24cafcaf4..708d1de81 100644 --- a/docs/categories/notes/page/4/index.html +++ b/docs/categories/notes/page/4/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/categories/notes/page/5/index.html b/docs/categories/notes/page/5/index.html index 3ae2abd2a..c8172b6a0 100644 --- a/docs/categories/notes/page/5/index.html +++ b/docs/categories/notes/page/5/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/categories/notes/page/6/index.html b/docs/categories/notes/page/6/index.html index 5c69ecdb4..398fc9f0b 100644 --- a/docs/categories/notes/page/6/index.html +++ b/docs/categories/notes/page/6/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/index.html b/docs/index.html index 0bab9433b..1c4566c67 100644 --- a/docs/index.html +++ b/docs/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/page/2/index.html b/docs/page/2/index.html index 48fb603ea..832db48ba 100644 --- a/docs/page/2/index.html +++ b/docs/page/2/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/page/3/index.html b/docs/page/3/index.html index f3d3c5a60..c1dd5b4c8 100644 --- a/docs/page/3/index.html +++ b/docs/page/3/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/page/4/index.html b/docs/page/4/index.html index 7e9c40e2c..d86d70de0 100644 --- a/docs/page/4/index.html +++ b/docs/page/4/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/page/5/index.html b/docs/page/5/index.html index f2ee5448c..a5c8d6273 100644 --- a/docs/page/5/index.html +++ b/docs/page/5/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/page/6/index.html b/docs/page/6/index.html index 7e405d77a..6f3dc3862 100644 --- a/docs/page/6/index.html +++ b/docs/page/6/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/page/7/index.html b/docs/page/7/index.html index b0ad648a7..c674752d3 100644 --- a/docs/page/7/index.html +++ b/docs/page/7/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/page/8/index.html b/docs/page/8/index.html index f149a76fd..a38f6032a 100644 --- a/docs/page/8/index.html +++ b/docs/page/8/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/posts/index.html b/docs/posts/index.html index 8b2124d96..67330b610 100644 --- a/docs/posts/index.html +++ b/docs/posts/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/posts/page/2/index.html b/docs/posts/page/2/index.html index f00ac006d..d83572472 100644 --- a/docs/posts/page/2/index.html +++ b/docs/posts/page/2/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/posts/page/3/index.html b/docs/posts/page/3/index.html index 2524552a8..62253cb5c 100644 --- a/docs/posts/page/3/index.html +++ b/docs/posts/page/3/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/posts/page/4/index.html b/docs/posts/page/4/index.html index 50a4b63fc..5a71274a5 100644 --- a/docs/posts/page/4/index.html +++ b/docs/posts/page/4/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/posts/page/5/index.html b/docs/posts/page/5/index.html index 7efa15079..b4b2291da 100644 --- a/docs/posts/page/5/index.html +++ b/docs/posts/page/5/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/posts/page/6/index.html b/docs/posts/page/6/index.html index b85a1dd02..704920c54 100644 --- a/docs/posts/page/6/index.html +++ b/docs/posts/page/6/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/posts/page/7/index.html b/docs/posts/page/7/index.html index d85b909e2..31c76bfa3 100644 --- a/docs/posts/page/7/index.html +++ b/docs/posts/page/7/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/posts/page/8/index.html b/docs/posts/page/8/index.html index b196fbb6f..ac3c75694 100644 --- a/docs/posts/page/8/index.html +++ b/docs/posts/page/8/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/sitemap.xml b/docs/sitemap.xml index 781eb4f17..b30dfbc9a 100644 --- a/docs/sitemap.xml +++ b/docs/sitemap.xml @@ -3,19 +3,19 @@ xmlns:xhtml="http://www.w3.org/1999/xhtml"> https://alanorth.github.io/cgspace-notes/categories/ - 2021-10-17T20:47:01+03:00 + 2021-10-20T22:21:55+03:00 https://alanorth.github.io/cgspace-notes/ - 2021-10-17T20:47:01+03:00 + 2021-10-20T22:21:55+03:00 https://alanorth.github.io/cgspace-notes/categories/notes/ - 2021-10-17T20:47:01+03:00 + 2021-10-20T22:21:55+03:00 https://alanorth.github.io/cgspace-notes/2021-10/ - 2021-10-17T20:47:01+03:00 + 2021-10-20T22:21:55+03:00 https://alanorth.github.io/cgspace-notes/posts/ - 2021-10-17T20:47:01+03:00 + 2021-10-20T22:21:55+03:00 https://alanorth.github.io/cgspace-notes/2021-09/ 2021-10-04T11:10:54+03:00