From dded1179fac3bc73c4d0aa8fd0794f9ef040578d Mon Sep 17 00:00:00 2001
From: Alan Orth <alan.orth@gmail.com>
Date: Thu, 31 Aug 2017 22:40:57 +0700
Subject: [PATCH] Update notes for 2017-08-31

---
 content/post/2017-08.md   | 10 ++++++++++
 public/2017-08/index.html | 17 ++++++++++++++---
 public/sitemap.xml        | 10 +++++-----
 3 files changed, 29 insertions(+), 8 deletions(-)

diff --git a/content/post/2017-08.md b/content/post/2017-08.md
index a29d57f07..46a44d1b8 100644
--- a/content/post/2017-08.md
+++ b/content/post/2017-08.md
@@ -375,3 +375,13 @@ dspace=# select handle from item, handle where handle.resource_id = item.item_id
 - I notice that in many WLE collections Marianne Gadeberg is in the edit or approval steps, but she is also in the groups for those steps.
 - I think we need to have a process to go back and check / fix some of these scenarios—to remove her user from the step and instead add her to the group—because we have way too many authorizations and in late 2016 we had [performance issues with Solr](https://github.com/ilri/rmg-ansible-public/commit/358b5ea43f9e5820986f897c9d560937c702ac6e) because of this
 - I asked Sisay about this and hinted that he should go back and fix these things, but let's see what he says
+- Saw CGSpace go down briefly today and noticed SQL connection pool errors in the dspace log file:
+
+```
+ERROR org.dspace.storage.rdbms.DatabaseManager @ SQL connection Error
+org.apache.commons.dbcp.SQLNestedException: Cannot get a connection, pool error Timeout waiting for idle object
+```
+
+- Looking at the logs I see we have been having hundreds or thousands of these errors a few times per week in 2017-07 and almost every day in 2017-08
+- It seems that I changed the `db.maxconnections` setting from 70 to 40 around 2017-08-14, but Macaroni Bros also reduced their hourly hammering of the REST API then
+- Nevertheless, it seems like a connection limit is not enough and that I should increase it (as well as the system's PostgreSQL `max_connections`
diff --git a/public/2017-08/index.html b/public/2017-08/index.html
index fdbeb624e..598eaa7cb 100644
--- a/public/2017-08/index.html
+++ b/public/2017-08/index.html
@@ -37,7 +37,7 @@ Then I cleaned up the author authorities and HTML characters in OpenRefine and s
 
 
 <meta property="article:published_time" content="2017-08-01T11:51:52&#43;03:00"/>
-<meta property="article:modified_time" content="2017-08-29T00:24:05&#43;07:00"/>
+<meta property="article:modified_time" content="2017-08-31T10:35:17&#43;07:00"/>
 
 
 
@@ -85,9 +85,9 @@ Then I cleaned up the author authorities and HTML characters in OpenRefine and s
   "@type": "BlogPosting",
   "headline": "August, 2017",
   "url": "https://alanorth.github.io/cgspace-notes/2017-08/",
-  "wordCount": "3428",
+  "wordCount": "3542",
   "datePublished": "2017-08-01T11:51:52&#43;03:00",
-  "dateModified": "2017-08-29T00:24:05&#43;07:00",
+  "dateModified": "2017-08-31T10:35:17&#43;07:00",
   "author": {
     "@type": "Person",
     "name": "Alan Orth"
@@ -589,6 +589,17 @@ WHERE {
 <li>I notice that in many WLE collections Marianne Gadeberg is in the edit or approval steps, but she is also in the groups for those steps.</li>
 <li>I think we need to have a process to go back and check / fix some of these scenarios—to remove her user from the step and instead add her to the group—because we have way too many authorizations and in late 2016 we had <a href="https://github.com/ilri/rmg-ansible-public/commit/358b5ea43f9e5820986f897c9d560937c702ac6e">performance issues with Solr</a> because of this</li>
 <li>I asked Sisay about this and hinted that he should go back and fix these things, but let&rsquo;s see what he says</li>
+<li>Saw CGSpace go down briefly today and noticed SQL connection pool errors in the dspace log file:</li>
+</ul>
+
+<pre><code>ERROR org.dspace.storage.rdbms.DatabaseManager @ SQL connection Error
+org.apache.commons.dbcp.SQLNestedException: Cannot get a connection, pool error Timeout waiting for idle object
+</code></pre>
+
+<ul>
+<li>Looking at the logs I see we have been having hundreds or thousands of these errors a few times per week in 2017-07 and almost every day in 2017-08</li>
+<li>It seems that I changed the <code>db.maxconnections</code> setting from 70 to 40 around 2017-08-14, but Macaroni Bros also reduced their hourly hammering of the REST API then</li>
+<li>Nevertheless, it seems like a connection limit is not enough and that I should increase it (as well as the system&rsquo;s PostgreSQL <code>max_connections</code></li>
 </ul>
 
   
diff --git a/public/sitemap.xml b/public/sitemap.xml
index 82d061ecb..926ce7969 100644
--- a/public/sitemap.xml
+++ b/public/sitemap.xml
@@ -4,7 +4,7 @@
   
   <url>
     <loc>https://alanorth.github.io/cgspace-notes/2017-08/</loc>
-    <lastmod>2017-08-29T00:24:05+07:00</lastmod>
+    <lastmod>2017-08-31T10:35:17+07:00</lastmod>
   </url>
   
   <url>
@@ -114,7 +114,7 @@
   
   <url>
     <loc>https://alanorth.github.io/cgspace-notes/</loc>
-    <lastmod>2017-08-29T00:24:05+07:00</lastmod>
+    <lastmod>2017-08-31T10:35:17+07:00</lastmod>
     <priority>0</priority>
   </url>
   
@@ -125,19 +125,19 @@
   
   <url>
     <loc>https://alanorth.github.io/cgspace-notes/tags/notes/</loc>
-    <lastmod>2017-08-29T00:24:05+07:00</lastmod>
+    <lastmod>2017-08-31T10:35:17+07:00</lastmod>
     <priority>0</priority>
   </url>
   
   <url>
     <loc>https://alanorth.github.io/cgspace-notes/post/</loc>
-    <lastmod>2017-08-29T00:24:05+07:00</lastmod>
+    <lastmod>2017-08-31T10:35:17+07:00</lastmod>
     <priority>0</priority>
   </url>
   
   <url>
     <loc>https://alanorth.github.io/cgspace-notes/tags/</loc>
-    <lastmod>2017-08-29T00:24:05+07:00</lastmod>
+    <lastmod>2017-08-31T10:35:17+07:00</lastmod>
     <priority>0</priority>
   </url>