From b23226a46d5fc41dec9adf4bc6c53663625b09df Mon Sep 17 00:00:00 2001
From: Alan Orth <alan.orth@gmail.com>
Date: Wed, 29 Nov 2017 17:26:17 +0300
Subject: [PATCH] Update notes for 2017-11-29

---
 content/post/2017-11.md   | 21 +++++++++++++++++++++
 public/2017-11/index.html | 29 ++++++++++++++++++++++++++---
 public/sitemap.xml        | 10 +++++-----
 3 files changed, 52 insertions(+), 8 deletions(-)

diff --git a/content/post/2017-11.md b/content/post/2017-11.md
index 6444b5a41..d5bbb7075 100644
--- a/content/post/2017-11.md
+++ b/content/post/2017-11.md
@@ -893,3 +893,24 @@ $ grep 70.32.83.92 dspace.log.2017-11-23 | grep -o -E 'session_id=[A-Z0-9]{32}'
 
 - PostgreSQL activity shows 69 connections
 - I don't have time to troubleshoot more as I'm in Nairobi working on the HPC so I just restarted Tomcat for now
+- A few hours later Uptime Robot says the server is down again
+- I don't see much activity in the logs but there are 87 PostgreSQL connections
+- But shit, there were 10,000 unique Tomcat sessions today:
+
+```
+$ cat dspace.log.2017-11-29 | grep -o -E 'session_id=[A-Z0-9]{32}' | sort -n | uniq | wc -l
+10037
+```
+
+- Although maybe that's not much, as the previous two days had more:
+
+```
+$ cat dspace.log.2017-11-27 | grep -o -E 'session_id=[A-Z0-9]{32}' | sort -n | uniq | wc -l
+12377
+$ cat dspace.log.2017-11-28 | grep -o -E 'session_id=[A-Z0-9]{32}' | sort -n | uniq | wc -l
+16984
+```
+
+- I think we just need start increasing the number of allowed PostgreSQL connections instead of fighting this, as it's the most common source of crashes we have
+- I will bump DSpace's `db.maxconnections` from 60 to 90, and PostgreSQL's `max_connections` from 183 to 273 (which is using my loose formula of 90 * webapps + 3)
+- I really need to figure out how to get DSpace to use a PostgreSQL connection pool
diff --git a/public/2017-11/index.html b/public/2017-11/index.html
index bbf0e0e96..0850f9ebd 100644
--- a/public/2017-11/index.html
+++ b/public/2017-11/index.html
@@ -38,7 +38,7 @@ COPY 54701
 
 <meta property="article:published_time" content="2017-11-02T09:37:54&#43;02:00"/>
 
-<meta property="article:modified_time" content="2017-11-26T21:26:13&#43;03:00"/>
+<meta property="article:modified_time" content="2017-11-29T10:14:09&#43;03:00"/>
 
 
 
@@ -86,9 +86,9 @@ COPY 54701
   "@type": "BlogPosting",
   "headline": "November, 2017",
   "url": "https://alanorth.github.io/cgspace-notes/2017-11/",
-  "wordCount": "5220",
+  "wordCount": "5389",
   "datePublished": "2017-11-02T09:37:54&#43;02:00",
-  "dateModified": "2017-11-26T21:26:13&#43;03:00",
+  "dateModified": "2017-11-29T10:14:09&#43;03:00",
   "author": {
     "@type": "Person",
     "name": "Alan Orth"
@@ -1152,6 +1152,29 @@ $ grep -c com.atmire.utils.UpdateSolrStatsMetadata dspace.log.2017-11-19
 <ul>
 <li>PostgreSQL activity shows 69 connections</li>
 <li>I don&rsquo;t have time to troubleshoot more as I&rsquo;m in Nairobi working on the HPC so I just restarted Tomcat for now</li>
+<li>A few hours later Uptime Robot says the server is down again</li>
+<li>I don&rsquo;t see much activity in the logs but there are 87 PostgreSQL connections</li>
+<li>But shit, there were 10,000 unique Tomcat sessions today:</li>
+</ul>
+
+<pre><code>$ cat dspace.log.2017-11-29 | grep -o -E 'session_id=[A-Z0-9]{32}' | sort -n | uniq | wc -l
+10037
+</code></pre>
+
+<ul>
+<li>Although maybe that&rsquo;s not much, as the previous two days had more:</li>
+</ul>
+
+<pre><code>$ cat dspace.log.2017-11-27 | grep -o -E 'session_id=[A-Z0-9]{32}' | sort -n | uniq | wc -l
+12377
+$ cat dspace.log.2017-11-28 | grep -o -E 'session_id=[A-Z0-9]{32}' | sort -n | uniq | wc -l
+16984
+</code></pre>
+
+<ul>
+<li>I think we just need start increasing the number of allowed PostgreSQL connections instead of fighting this, as it&rsquo;s the most common source of crashes we have</li>
+<li>I will bump DSpace&rsquo;s <code>db.maxconnections</code> from 60 to 90, and PostgreSQL&rsquo;s <code>max_connections</code> from 183 to 273 (which is using my loose formula of 90 * webapps + 3)</li>
+<li>I really need to figure out how to get DSpace to use a PostgreSQL connection pool</li>
 </ul>
 
   
diff --git a/public/sitemap.xml b/public/sitemap.xml
index 67509515e..4c9ba4fba 100644
--- a/public/sitemap.xml
+++ b/public/sitemap.xml
@@ -4,7 +4,7 @@
   
   <url>
     <loc>https://alanorth.github.io/cgspace-notes/2017-11/</loc>
-    <lastmod>2017-11-26T21:26:13+03:00</lastmod>
+    <lastmod>2017-11-29T10:14:09+03:00</lastmod>
   </url>
   
   <url>
@@ -134,7 +134,7 @@
   
   <url>
     <loc>https://alanorth.github.io/cgspace-notes/</loc>
-    <lastmod>2017-11-26T21:26:13+03:00</lastmod>
+    <lastmod>2017-11-29T10:14:09+03:00</lastmod>
     <priority>0</priority>
   </url>
   
@@ -145,7 +145,7 @@
   
   <url>
     <loc>https://alanorth.github.io/cgspace-notes/tags/notes/</loc>
-    <lastmod>2017-11-26T21:26:13+03:00</lastmod>
+    <lastmod>2017-11-29T10:14:09+03:00</lastmod>
     <priority>0</priority>
   </url>
   
@@ -157,13 +157,13 @@
   
   <url>
     <loc>https://alanorth.github.io/cgspace-notes/post/</loc>
-    <lastmod>2017-11-26T21:26:13+03:00</lastmod>
+    <lastmod>2017-11-29T10:14:09+03:00</lastmod>
     <priority>0</priority>
   </url>
   
   <url>
     <loc>https://alanorth.github.io/cgspace-notes/tags/</loc>
-    <lastmod>2017-11-26T21:26:13+03:00</lastmod>
+    <lastmod>2017-11-29T10:14:09+03:00</lastmod>
     <priority>0</priority>
   </url>