diff --git a/content/posts/2019-04.md b/content/posts/2019-04.md index acc096946..5a9b00408 100644 --- a/content/posts/2019-04.md +++ b/content/posts/2019-04.md @@ -507,4 +507,51 @@ org.apache.tomcat.jdbc.pool.PoolExhaustedException: [http-bio-127.0.0.1-8443-exe 370 51.254.16.223 ``` +## 2019-04-09 + +- Linode sent an alert that CGSpace (linode18) was 440% CPU for the last two hours this morning +- Here are the top IPs in the web server logs around that time: + +``` +# zcat --force /var/log/nginx/{rest,oai}.log /var/log/nginx/{rest,oai}.log.1 | grep -E "09/Apr/2019:(06|07|08)" | awk '{print $1}' | sort | uniq -c | sort -n | tail -n 10 + 18 66.249.79.139 + 21 157.55.39.160 + 29 66.249.79.137 + 38 66.249.79.135 + 50 34.200.212.137 + 54 66.249.79.133 + 100 102.128.190.18 + 1166 45.5.184.72 + 4251 45.5.186.2 + 4895 205.186.128.185 +# zcat --force /var/log/nginx/{access,library-access}.log /var/log/nginx/{access,library-access}.log.1 | grep -E "09/Apr/2019:(06|07|08)" | awk '{print $1}' | sort | uniq -c | sort -n | tail -n 10 + 200 144.48.242.108 + 202 207.46.13.185 + 206 18.194.46.84 + 239 66.249.79.139 + 246 18.196.196.108 + 274 31.6.77.23 + 289 66.249.79.137 + 312 157.55.39.160 + 441 66.249.79.135 + 856 66.249.79.133 +``` + +- `45.5.186.2` is at CIAT in Colombia and I see they are mostly making requests to the REST API, but also to XMLUI with the following user agent: + +``` +Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36 +``` + +- Database connection usage looks fine: + +``` +$ psql -c 'select * from pg_stat_activity' | grep -o -E '(dspaceWeb|dspaceApi|dspaceCli)' | sort | uniq -c + 5 dspaceApi + 7 dspaceCli + 11 dspaceWeb +``` + +- Ironically I do still see some 2 to 10% of CPU steal in `iostat 1 10` + diff --git a/docs/2019-04/index.html b/docs/2019-04/index.html index 8e5dfb63a..25d8fba8b 100644 --- a/docs/2019-04/index.html +++ b/docs/2019-04/index.html @@ -38,7 +38,7 @@ $ ./delete-metadata-values.py -i /tmp/2019-02-21-delete-1-region.csv -db dspace - + @@ -81,9 +81,9 @@ $ ./delete-metadata-values.py -i /tmp/2019-02-21-delete-1-region.csv -db dspace "@type": "BlogPosting", "headline": "April, 2019", "url": "https://alanorth.github.io/cgspace-notes/2019-04/", - "wordCount": "2869", + "wordCount": "3073", "datePublished": "2019-04-01T09:00:43+03:00", - "dateModified": "2019-04-08T20:03:58+03:00", + "dateModified": "2019-04-08T20:22:40+03:00", "author": { "@type": "Person", "name": "Alan Orth" @@ -750,6 +750,58 @@ org.apache.tomcat.jdbc.pool.PoolExhaustedException: [http-bio-127.0.0.1-8443-exe 370 51.254.16.223 +

2019-04-09

+ + + +
# zcat --force /var/log/nginx/{rest,oai}.log /var/log/nginx/{rest,oai}.log.1 | grep -E "09/Apr/2019:(06|07|08)" | awk '{print $1}' | sort | uniq -c | sort -n | tail -n 10
+     18 66.249.79.139
+     21 157.55.39.160
+     29 66.249.79.137
+     38 66.249.79.135
+     50 34.200.212.137
+     54 66.249.79.133
+    100 102.128.190.18
+   1166 45.5.184.72
+   4251 45.5.186.2
+   4895 205.186.128.185
+# zcat --force /var/log/nginx/{access,library-access}.log /var/log/nginx/{access,library-access}.log.1 | grep -E "09/Apr/2019:(06|07|08)" | awk '{print $1}' | sort | uniq -c | sort -n | tail -n 10
+    200 144.48.242.108
+    202 207.46.13.185
+    206 18.194.46.84
+    239 66.249.79.139
+    246 18.196.196.108
+    274 31.6.77.23
+    289 66.249.79.137
+    312 157.55.39.160
+    441 66.249.79.135
+    856 66.249.79.133
+
+ + + +
Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36
+
+ + + +
$ psql -c 'select * from pg_stat_activity' | grep -o -E '(dspaceWeb|dspaceApi|dspaceCli)' | sort | uniq -c
+      5 dspaceApi
+      7 dspaceCli
+     11 dspaceWeb
+
+ + + diff --git a/docs/sitemap.xml b/docs/sitemap.xml index e8ef340ad..f95d8b2f2 100644 --- a/docs/sitemap.xml +++ b/docs/sitemap.xml @@ -4,7 +4,7 @@ https://alanorth.github.io/cgspace-notes/2019-04/ - 2019-04-08T20:03:58+03:00 + 2019-04-08T20:22:40+03:00 @@ -219,7 +219,7 @@ https://alanorth.github.io/cgspace-notes/ - 2019-04-08T20:03:58+03:00 + 2019-04-08T20:22:40+03:00 0 @@ -230,7 +230,7 @@ https://alanorth.github.io/cgspace-notes/tags/notes/ - 2019-04-08T20:03:58+03:00 + 2019-04-08T20:22:40+03:00 0 @@ -242,13 +242,13 @@ https://alanorth.github.io/cgspace-notes/posts/ - 2019-04-08T20:03:58+03:00 + 2019-04-08T20:22:40+03:00 0 https://alanorth.github.io/cgspace-notes/tags/ - 2019-04-08T20:03:58+03:00 + 2019-04-08T20:22:40+03:00 0