From c8cc46b6f6abe70b36dabf2125079126f2accf37 Mon Sep 17 00:00:00 2001 From: Alan Orth Date: Thu, 4 Jan 2018 08:02:36 -0800 Subject: [PATCH] Add notes for 2018-01-04 --- content/post/2018-01.md | 39 ++++++++++++++++++++++++++++++ public/2018-01/index.html | 50 ++++++++++++++++++++++++++++++++++++--- public/sitemap.xml | 10 ++++---- 3 files changed, 91 insertions(+), 8 deletions(-) diff --git a/content/post/2018-01.md b/content/post/2018-01.md index ca9bcf7c1..1b4bb5c8e 100644 --- a/content/post/2018-01.md +++ b/content/post/2018-01.md @@ -185,3 +185,42 @@ $ grep 134.155.96.78 dspace.log.2018-01-03 | grep -o -E 'session_id=[A-Z0-9]{32} - I have no idea what these are but they seem to be coming from Amazon... - I guess for now I just have to increase the database connection pool's max active - It's currently 75 and normally I'd just bump it by 25 but let me be a bit daring and push it by 50 to 125, because I used to see at least 121 connections in pg_stat_activity before when we were using the shitty default pooling + +## 2018-01-04 + +- CGSpace went down and up a bunch of times last night and ILRI staff were complaining a lot last night +- The XMLUI logs show this activity: + +``` +# cat /var/log/nginx/access.log /var/log/nginx/access.log.1 /var/log/nginx/library-access.log /var/log/nginx/library-access.log.1 | grep -E "4/Jan/2018" | awk '{print $1}' | sort -n | uniq -c | sort -h | tail + 968 197.211.63.81 + 981 213.55.99.121 + 1039 66.249.64.93 + 1258 157.55.39.175 + 1273 207.46.13.182 + 1311 157.55.39.191 + 1319 157.55.39.197 + 1775 66.249.64.78 + 2216 104.196.152.243 + 3366 66.249.64.91 +``` + +- Again we ran out of PostgreSQL database connections, even after bumping the pool max active limit from 50 to 75 to 125 yesterday! + +``` +2018-01-04 07:36:08,089 ERROR org.dspace.storage.rdbms.DatabaseManager @ SQL connection Error - +org.apache.tomcat.jdbc.pool.PoolExhaustedException: [http-bio-127.0.0.1-8443-exec-256] Timeout: Pool empty. Unable to fetch a connection in 5 seconds, none available[size:125; busy:125; idle:0; lastwait:5000]. +``` + +- So for this week that is the number one problem! + +``` +$ grep -c "Timeout: Pool empty." dspace.log.2018-01-* +dspace.log.2018-01-01:0 +dspace.log.2018-01-02:1972 +dspace.log.2018-01-03:1909 +dspace.log.2018-01-04:1559 +``` + +- I will just bump the connection limit to 300 because I'm fucking fed up with this shit +- Once I get back to Amman I will have to try to create different database pools for different web applications, like recently discussed on the dspace-tech mailing list diff --git a/public/2018-01/index.html b/public/2018-01/index.html index 726495630..242fc2611 100644 --- a/public/2018-01/index.html +++ b/public/2018-01/index.html @@ -92,7 +92,7 @@ Danny wrote to ask for help renewing the wildcard ilri.org certificate and I adv - + @@ -194,9 +194,9 @@ Danny wrote to ask for help renewing the wildcard ilri.org certificate and I adv "@type": "BlogPosting", "headline": "January, 2018", "url": "https://alanorth.github.io/cgspace-notes/2018-01/", - "wordCount": "731", + "wordCount": "919", "datePublished": "2018-01-02T08:35:54-08:00", - "dateModified": "2018-01-02T09:30:34-08:00", + "dateModified": "2018-01-03T07:49:38-08:00", "author": { "@type": "Person", "name": "Alan Orth" @@ -455,6 +455,50 @@ dspace.log.2018-01-03:1909
  • It’s currently 75 and normally I’d just bump it by 25 but let me be a bit daring and push it by 50 to 125, because I used to see at least 121 connections in pg_stat_activity before when we were using the shitty default pooling
    • +

    2018-01-04

    + + + +
    # cat /var/log/nginx/access.log /var/log/nginx/access.log.1 /var/log/nginx/library-access.log /var/log/nginx/library-access.log.1 | grep -E "4/Jan/2018" | awk '{print $1}' | sort -n | uniq -c | sort -h | tail
+    968 197.211.63.81
+    981 213.55.99.121
+   1039 66.249.64.93
+   1258 157.55.39.175
+   1273 207.46.13.182
+   1311 157.55.39.191
+   1319 157.55.39.197
+   1775 66.249.64.78
+   2216 104.196.152.243
+   3366 66.249.64.91
+
    + + + +
    2018-01-04 07:36:08,089 ERROR org.dspace.storage.rdbms.DatabaseManager @ SQL connection Error -
+org.apache.tomcat.jdbc.pool.PoolExhaustedException: [http-bio-127.0.0.1-8443-exec-256] Timeout: Pool empty. Unable to fetch a connection in 5 seconds, none available[size:125; busy:125; idle:0; lastwait:5000].
+
    + + + +
    $ grep -c "Timeout: Pool empty." dspace.log.2018-01-*
+dspace.log.2018-01-01:0
+dspace.log.2018-01-02:1972
+dspace.log.2018-01-03:1909
+dspace.log.2018-01-04:1559
+
    + + + diff --git a/public/sitemap.xml b/public/sitemap.xml index e4233e2ef..d44ffee8d 100644 --- a/public/sitemap.xml +++ b/public/sitemap.xml @@ -4,7 +4,7 @@ https://alanorth.github.io/cgspace-notes/2018-01/ - 2018-01-02T09:30:34-08:00 + 2018-01-03T07:49:38-08:00 @@ -144,7 +144,7 @@ https://alanorth.github.io/cgspace-notes/ - 2018-01-02T09:30:34-08:00 + 2018-01-03T07:49:38-08:00 0 @@ -155,7 +155,7 @@ https://alanorth.github.io/cgspace-notes/tags/notes/ - 2018-01-02T09:30:34-08:00 + 2018-01-03T07:49:38-08:00 0 @@ -167,13 +167,13 @@ https://alanorth.github.io/cgspace-notes/post/ - 2018-01-02T09:30:34-08:00 + 2018-01-03T07:49:38-08:00 0 https://alanorth.github.io/cgspace-notes/tags/ - 2018-01-02T09:30:34-08:00 + 2018-01-03T07:49:38-08:00 0