From 7e2681c0f7f7886363f50ecb274fb6ba6c65d05b Mon Sep 17 00:00:00 2001 From: Alan Orth Date: Tue, 10 Apr 2018 11:33:36 +0300 Subject: [PATCH] Update notes for 2018-04-10 --- content/posts/2018-04.md | 57 ++++++++++++++++++++++++++++++++- docs/2017-11/index.html | 4 +-- docs/2018-04/index.html | 69 +++++++++++++++++++++++++++++++++++++--- docs/sitemap.xml | 12 +++---- 4 files changed, 129 insertions(+), 13 deletions(-) diff --git a/content/posts/2018-04.md b/content/posts/2018-04.md index 1a5b39394..c98b66bbe 100644 --- a/content/posts/2018-04.md +++ b/content/posts/2018-04.md @@ -210,7 +210,7 @@ X-XSS-Protection: 1; mode=block ![Tomcat sessions week](/cgspace-notes/2018/04/jmx_dspace_sessions-week.png) -- Looks like the number of total requests processed by nginx in March went down from the previous months: +- In other news, it looks like the number of total requests processed by nginx in March went down from the previous months: ``` # time zcat --force /var/log/nginx/* | grep -cE "[0-9]{1,2}/Mar/2018" @@ -220,3 +220,58 @@ real 0m13.658s user 0m16.533s sys 0m1.087s ``` + +- In other other news, the database cleanup script has an issue again: + +``` +$ dspace cleanup -v +... +Error: ERROR: update or delete on table "bitstream" violates foreign key constraint "bundle_primary_bitstream_id_fkey" on table "bundle" + Detail: Key (bitstream_id)=(151626) is still referenced from table "bundle". +``` + +- The solution is, as always: + +``` +$ psql dspace -c 'update bundle set primary_bitstream_id=NULL where primary_bitstream_id in (151626);' +UPDATE 1 +``` + +- Looking at abandoned connections in Tomcat: + +``` +# zcat /var/log/tomcat7/catalina.out.[1-9].gz | grep -c 'org.apache.tomcat.jdbc.pool.ConnectionPool abandon' +2115 +``` + +- Apparently from these stacktraces we should be able to see which code is not closing connections properly +- Here's a pretty good overview of days where we had database issues recently: + +``` +# zcat /var/log/tomcat7/catalina.out.[1-9].gz | grep 'org.apache.tomcat.jdbc.pool.ConnectionPool abandon' | awk '{print $1,$2, $3}' | sort | uniq -c | sort -n + 1 Feb 18, 2018 + 1 Feb 19, 2018 + 1 Feb 20, 2018 + 1 Feb 24, 2018 + 2 Feb 13, 2018 + 3 Feb 17, 2018 + 5 Feb 16, 2018 + 5 Feb 23, 2018 + 5 Feb 27, 2018 + 6 Feb 25, 2018 + 40 Feb 14, 2018 + 63 Feb 28, 2018 + 154 Mar 19, 2018 + 202 Feb 21, 2018 + 264 Feb 26, 2018 + 268 Mar 21, 2018 + 524 Feb 22, 2018 + 570 Feb 15, 2018 +``` + +- In Tomcat 8.5 the `removeAbandoned` property has been split into two: `removeAbandonedOnBorrow` and `removeAbandonedOnMaintenance` +- See: https://tomcat.apache.org/tomcat-8.5-doc/jndi-datasource-examples-howto.html#Database_Connection_Pool_(DBCP_2)_Configurations +- I assume we want `removeAbandonedOnBorrow` and make updates to the Tomcat 8 templates in Ansible +- After reading more documentation I see that Tomcat 8.5's default DBCP seems to now be Commons DBCP2 instead of Tomcat DBCP +- It can be overridden in Tomcat's _server.xml_ by setting `factory="org.apache.tomcat.jdbc.pool.DataSourceFactory"` in the `` +- I think we should use this default, so we'll need to remove some other settings that are specific to Tomcat's DBCP like `jdbcInterceptors` and `abandonWhenPercentageFull` diff --git a/docs/2017-11/index.html b/docs/2017-11/index.html index 9ef5651d0..f09f9571a 100644 --- a/docs/2017-11/index.html +++ b/docs/2017-11/index.html @@ -38,7 +38,7 @@ COPY 54701 - + @@ -89,7 +89,7 @@ COPY 54701 "url": "https://alanorth.github.io/cgspace-notes/2017-11/", "wordCount": "5428", "datePublished": "2017-11-02T09:37:54+02:00", - "dateModified": "2018-03-09T22:10:33+02:00", + "dateModified": "2018-04-10T08:27:55+03:00", "author": { "@type": "Person", "name": "Alan Orth" diff --git a/docs/2018-04/index.html b/docs/2018-04/index.html index 698fdbfad..d9843cc3c 100644 --- a/docs/2018-04/index.html +++ b/docs/2018-04/index.html @@ -21,7 +21,7 @@ Catalina logs at least show some memory errors yesterday: - + @@ -53,9 +53,9 @@ Catalina logs at least show some memory errors yesterday: "@type": "BlogPosting", "headline": "April, 2018", "url": "https://alanorth.github.io/cgspace-notes/2018-04/", - "wordCount": "1005", + "wordCount": "1297", "datePublished": "2018-04-01T16:13:54+02:00", - "dateModified": "2018-04-04T17:01:08+03:00", + "dateModified": "2018-04-10T08:27:55+03:00", "author": { "@type": "Person", "name": "Alan Orth" @@ -351,7 +351,7 @@ X-XSS-Protection: 1; mode=block

Tomcat sessions week

# time zcat --force /var/log/nginx/* | grep -cE "[0-9]{1,2}/Mar/2018"
@@ -362,6 +362,67 @@ user    0m16.533s
 sys     0m1.087s
+ + +
$ dspace cleanup -v
+...
+Error: ERROR: update or delete on table "bitstream" violates foreign key constraint "bundle_primary_bitstream_id_fkey" on table "bundle"
+  Detail: Key (bitstream_id)=(151626) is still referenced from table "bundle".
+
+ + + +
$ psql dspace -c 'update bundle set primary_bitstream_id=NULL where primary_bitstream_id in (151626);'
+UPDATE 1
+
+ + + +
# zcat /var/log/tomcat7/catalina.out.[1-9].gz | grep -c 'org.apache.tomcat.jdbc.pool.ConnectionPool abandon'
+2115
+
+ + + +
# zcat /var/log/tomcat7/catalina.out.[1-9].gz | grep 'org.apache.tomcat.jdbc.pool.ConnectionPool abandon' | awk '{print $1,$2, $3}' | sort | uniq -c | sort -n
+      1 Feb 18, 2018
+      1 Feb 19, 2018
+      1 Feb 20, 2018
+      1 Feb 24, 2018
+      2 Feb 13, 2018
+      3 Feb 17, 2018
+      5 Feb 16, 2018
+      5 Feb 23, 2018
+      5 Feb 27, 2018
+      6 Feb 25, 2018
+     40 Feb 14, 2018
+     63 Feb 28, 2018
+    154 Mar 19, 2018
+    202 Feb 21, 2018
+    264 Feb 26, 2018
+    268 Mar 21, 2018
+    524 Feb 22, 2018
+    570 Feb 15, 2018
+
+ + + diff --git a/docs/sitemap.xml b/docs/sitemap.xml index 7d8e06ab1..19281740d 100644 --- a/docs/sitemap.xml +++ b/docs/sitemap.xml @@ -4,7 +4,7 @@ https://alanorth.github.io/cgspace-notes/2018-04/ - 2018-04-04T17:01:08+03:00 + 2018-04-10T08:27:55+03:00 @@ -29,7 +29,7 @@ https://alanorth.github.io/cgspace-notes/2017-11/ - 2018-03-09T22:10:33+02:00 + 2018-04-10T08:27:55+03:00 @@ -159,7 +159,7 @@ https://alanorth.github.io/cgspace-notes/ - 2018-04-04T17:01:08+03:00 + 2018-04-10T08:27:55+03:00 0 @@ -170,7 +170,7 @@ https://alanorth.github.io/cgspace-notes/tags/notes/ - 2018-04-04T17:01:08+03:00 + 2018-04-10T08:27:55+03:00 0 @@ -182,13 +182,13 @@ https://alanorth.github.io/cgspace-notes/posts/ - 2018-04-04T17:01:08+03:00 + 2018-04-10T08:27:55+03:00 0 https://alanorth.github.io/cgspace-notes/tags/ - 2018-04-04T17:01:08+03:00 + 2018-04-10T08:27:55+03:00 0