diff --git a/content/posts/2019-04.md b/content/posts/2019-04.md index acc096946..5a9b00408 100644 --- a/content/posts/2019-04.md +++ b/content/posts/2019-04.md @@ -507,4 +507,51 @@ org.apache.tomcat.jdbc.pool.PoolExhaustedException: [http-bio-127.0.0.1-8443-exe 370 51.254.16.223 ``` +## 2019-04-09 + +- Linode sent an alert that CGSpace (linode18) was 440% CPU for the last two hours this morning +- Here are the top IPs in the web server logs around that time: + +``` +# zcat --force /var/log/nginx/{rest,oai}.log /var/log/nginx/{rest,oai}.log.1 | grep -E "09/Apr/2019:(06|07|08)" | awk '{print $1}' | sort | uniq -c | sort -n | tail -n 10 + 18 66.249.79.139 + 21 157.55.39.160 + 29 66.249.79.137 + 38 66.249.79.135 + 50 34.200.212.137 + 54 66.249.79.133 + 100 102.128.190.18 + 1166 45.5.184.72 + 4251 45.5.186.2 + 4895 205.186.128.185 +# zcat --force /var/log/nginx/{access,library-access}.log /var/log/nginx/{access,library-access}.log.1 | grep -E "09/Apr/2019:(06|07|08)" | awk '{print $1}' | sort | uniq -c | sort -n | tail -n 10 + 200 144.48.242.108 + 202 207.46.13.185 + 206 18.194.46.84 + 239 66.249.79.139 + 246 18.196.196.108 + 274 31.6.77.23 + 289 66.249.79.137 + 312 157.55.39.160 + 441 66.249.79.135 + 856 66.249.79.133 +``` + +- `45.5.186.2` is at CIAT in Colombia and I see they are mostly making requests to the REST API, but also to XMLUI with the following user agent: + +``` +Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36 +``` + +- Database connection usage looks fine: + +``` +$ psql -c 'select * from pg_stat_activity' | grep -o -E '(dspaceWeb|dspaceApi|dspaceCli)' | sort | uniq -c + 5 dspaceApi + 7 dspaceCli + 11 dspaceWeb +``` + +- Ironically I do still see some 2 to 10% of CPU steal in `iostat 1 10` + diff --git a/docs/2019-04/index.html b/docs/2019-04/index.html index 8e5dfb63a..25d8fba8b 100644 --- a/docs/2019-04/index.html +++ b/docs/2019-04/index.html @@ -38,7 +38,7 @@ $ ./delete-metadata-values.py -i /tmp/2019-02-21-delete-1-region.csv -db dspace - + @@ -81,9 +81,9 @@ $ ./delete-metadata-values.py -i /tmp/2019-02-21-delete-1-region.csv -db dspace "@type": "BlogPosting", "headline": "April, 2019", "url": "https://alanorth.github.io/cgspace-notes/2019-04/", - "wordCount": "2869", + "wordCount": "3073", "datePublished": "2019-04-01T09:00:43+03:00", - "dateModified": "2019-04-08T20:03:58+03:00", + "dateModified": "2019-04-08T20:22:40+03:00", "author": { "@type": "Person", "name": "Alan Orth" @@ -750,6 +750,58 @@ org.apache.tomcat.jdbc.pool.PoolExhaustedException: [http-bio-127.0.0.1-8443-exe 370 51.254.16.223 +
# zcat --force /var/log/nginx/{rest,oai}.log /var/log/nginx/{rest,oai}.log.1 | grep -E "09/Apr/2019:(06|07|08)" | awk '{print $1}' | sort | uniq -c | sort -n | tail -n 10
+ 18 66.249.79.139
+ 21 157.55.39.160
+ 29 66.249.79.137
+ 38 66.249.79.135
+ 50 34.200.212.137
+ 54 66.249.79.133
+ 100 102.128.190.18
+ 1166 45.5.184.72
+ 4251 45.5.186.2
+ 4895 205.186.128.185
+# zcat --force /var/log/nginx/{access,library-access}.log /var/log/nginx/{access,library-access}.log.1 | grep -E "09/Apr/2019:(06|07|08)" | awk '{print $1}' | sort | uniq -c | sort -n | tail -n 10
+ 200 144.48.242.108
+ 202 207.46.13.185
+ 206 18.194.46.84
+ 239 66.249.79.139
+ 246 18.196.196.108
+ 274 31.6.77.23
+ 289 66.249.79.137
+ 312 157.55.39.160
+ 441 66.249.79.135
+ 856 66.249.79.133
+
+
+45.5.186.2
is at CIAT in Colombia and I see they are mostly making requests to the REST API, but also to XMLUI with the following user agent:Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36
+
+
+$ psql -c 'select * from pg_stat_activity' | grep -o -E '(dspaceWeb|dspaceApi|dspaceCli)' | sort | uniq -c
+ 5 dspaceApi
+ 7 dspaceCli
+ 11 dspaceWeb
+
+
+iostat 1 10