Add notes for 2019-01-16

content/posts/2019-01.md

@@ -358,4 +358,133 @@ $ time http --print h 'https://digitalarchive.worldfishcenter.org/rest/items?exp
    2482 45.5.186.2
 ```
 
+## 2019-01-16
+
+- Day three of CGSpace AReS meeting in Amman
+  - We discussed CGcore and decided some action points
+  - We discussed branding of AReS tool
+- Something happened to the Solr usage statistics on CGSpace
+  - I looked on the server and the Solr cores are there (56GB!), and I don't see any obvious errors in dmesg or anything
+  - I see that the server hasn't been rebooted in 26 days so I rebooted it
+- After reboot the Solr stats are still messed up in the Atmire Usage Stats module, it only shows 2019-01!
+
+![Solr stats fucked up](/cgspace-notes/2019/01/solr-stats-incorrect.png)
+
+- In the Solr admin UI I see the following error:
+
+```
+statistics-2018: org.apache.solr.common.SolrException:org.apache.solr.common.SolrException: Error opening new searcher
+```
+
+- Looking in the Solr log I see this:
+
+```
+2019-01-16 13:37:55,395 ERROR org.apache.solr.core.CoreContainer @ Error creating core [statistics-2018]: Error opening new searcher
+org.apache.solr.common.SolrException: Error opening new searcher
+    at org.apache.solr.core.SolrCore.<init>(SolrCore.java:873)
+    at org.apache.solr.core.SolrCore.<init>(SolrCore.java:646)
+    at org.apache.solr.core.CoreContainer.create(CoreContainer.java:491)
+    at org.apache.solr.core.CoreContainer.create(CoreContainer.java:466)
+    at org.apache.solr.handler.admin.CoreAdminHandler.handleCreateAction(CoreAdminHandler.java:575)
+    at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestInternal(CoreAdminHandler.java:199)
+    at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:188)
+    at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:135)
+    at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:729)
+    at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:258)
+    at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:207)
+    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
+    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
+    at org.dspace.solr.filters.LocalHostRestrictionFilter.doFilter(LocalHostRestrictionFilter.java:50)
+    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
+    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
+    at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:221)
+    at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:122)
+    at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:505)
+    at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:169)
+    at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103)
+    at org.apache.catalina.valves.CrawlerSessionManagerValve.invoke(CrawlerSessionManagerValve.java:180)
+    at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:956)
+    at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116)
+    at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:436)
+    at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1078)
+    at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:625)
+    at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:316)
+    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
+    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
+    at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
+    at java.lang.Thread.run(Thread.java:748)
+Caused by: org.apache.solr.common.SolrException: Error opening new searcher
+    at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:1565)
+    at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:1677)
+    at org.apache.solr.core.SolrCore.<init>(SolrCore.java:845)
+    ... 31 more
+Caused by: org.apache.lucene.store.LockObtainFailedException: Lock obtain timed out: NativeFSLock@/home/cgspace.cgiar.org/solr/statistics-2018/data/index/write.lock
+    at org.apache.lucene.store.Lock.obtain(Lock.java:89)
+    at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:753)
+    at org.apache.solr.update.SolrIndexWriter.<init>(SolrIndexWriter.java:77)
+    at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:64)
+    at org.apache.solr.update.DefaultSolrCoreState.createMainIndexWriter(DefaultSolrCoreState.java:279)
+    at org.apache.solr.update.DefaultSolrCoreState.getIndexWriter(DefaultSolrCoreState.java:111)
+    at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:1528)
+    ... 33 more
+2019-01-16 13:37:55,401 ERROR org.apache.solr.core.SolrCore @ org.apache.solr.common.SolrException: Error CREATEing SolrCore 'statistics-2018': Unable to create core [statistics-2018] Caused by: Lock obtain timed out: NativeFSLock@/home/cgspace.cgiar.org/solr/statistics-2018/data/index/write.lock
+    at org.apache.solr.handler.admin.CoreAdminHandler.handleCreateAction(CoreAdminHandler.java:613)
+    at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestInternal(CoreAdminHandler.java:199)
+    at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:188)
+    at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:135)
+    at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:729)
+    at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:258)
+    at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:207)
+    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
+    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
+    at org.dspace.solr.filters.LocalHostRestrictionFilter.doFilter(LocalHostRestrictionFilter.java:50)
+    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
+    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
+    at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:221)
+    at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:122)
+    at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:505)
+    at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:169)
+    at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103)
+    at org.apache.catalina.valves.CrawlerSessionManagerValve.invoke(CrawlerSessionManagerValve.java:180)
+    at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:956)
+    at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116)
+    at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:436)
+    at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1078)
+    at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:625)
+    at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:316)
+    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
+    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
+    at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
+    at java.lang.Thread.run(Thread.java:748)
+Caused by: org.apache.solr.common.SolrException: Unable to create core [statistics-2018]
+    at org.apache.solr.core.CoreContainer.create(CoreContainer.java:507)
+    at org.apache.solr.core.CoreContainer.create(CoreContainer.java:466)
+    at org.apache.solr.handler.admin.CoreAdminHandler.handleCreateAction(CoreAdminHandler.java:575)
+    ... 27 more
+Caused by: org.apache.solr.common.SolrException: Error opening new searcher
+    at org.apache.solr.core.SolrCore.<init>(SolrCore.java:873)
+    at org.apache.solr.core.SolrCore.<init>(SolrCore.java:646)
+    at org.apache.solr.core.CoreContainer.create(CoreContainer.java:491)
+    ... 29 more
+Caused by: org.apache.solr.common.SolrException: Error opening new searcher
+    at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:1565)
+    at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:1677)
+    at org.apache.solr.core.SolrCore.<init>(SolrCore.java:845)
+    ... 31 more
+Caused by: org.apache.lucene.store.LockObtainFailedException: Lock obtain timed out: NativeFSLock@/home/cgspace.cgiar.org/solr/statistics-2018/data/index/write.lock
+    at org.apache.lucene.store.Lock.obtain(Lock.java:89)
+    at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:753)
+    at org.apache.solr.update.SolrIndexWriter.<init>(SolrIndexWriter.java:77)
+    at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:64)
+    at org.apache.solr.update.DefaultSolrCoreState.createMainIndexWriter(DefaultSolrCoreState.java:279)
+    at org.apache.solr.update.DefaultSolrCoreState.getIndexWriter(DefaultSolrCoreState.java:111)
+    at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:1528)
+    ... 33 more
+```
+
+- I found some threads on StackOverflow etc discussing this and several suggested increasing the address space for the shell with ulimit
+- I added `ulimit -v unlimited` to the `/etc/default/tomcat7` and restarted Tomcat and now Solr is working again:
+
+![Solr stats working](/cgspace-notes/2019/01/solr-stats-incorrect.png)
+
 <!-- vim: set sw=2 ts=2: -->

docs/2019-01/index.html

@@ -27,7 +27,7 @@ I don’t see anything interesting in the web server logs around that time t
 " />
 <meta property="og:type" content="article" />
 <meta property="og:url" content="https://alanorth.github.io/cgspace-notes/2019-01/" /><meta property="article:published_time" content="2019-01-02T09:48:30&#43;02:00"/>
-<meta property="article:modified_time" content="2019-01-15T16:35:16&#43;02:00"/>
+<meta property="article:modified_time" content="2019-01-15T16:51:35&#43;02:00"/>
 
 <meta name="twitter:card" content="summary"/>
 <meta name="twitter:title" content="January, 2019"/>
@@ -60,9 +60,9 @@ I don&rsquo;t see anything interesting in the web server logs around that time t
   "@type": "BlogPosting",
   "headline": "January, 2019",
   "url": "https://alanorth.github.io/cgspace-notes/2019-01/",
-  "wordCount": "1581",
+  "wordCount": "2007",
   "datePublished": "2019-01-02T09:48:30&#43;02:00",
-  "dateModified": "2019-01-15T16:35:16&#43;02:00",
+  "dateModified": "2019-01-15T16:51:35&#43;02:00",
   "author": {
     "@type": "Person",
     "name": "Alan Orth"
@@ -529,6 +529,147 @@ In [14]: for row in result.fetchone():
    2482 45.5.186.2
 </code></pre>
 
+<h2 id="2019-01-16">2019-01-16</h2>
+
+<ul>
+<li>Day three of CGSpace AReS meeting in Amman
+
+<ul>
+<li>We discussed CGcore and decided some action points</li>
+<li>We discussed branding of AReS tool</li>
+</ul></li>
+<li>Something happened to the Solr usage statistics on CGSpace
+
+<ul>
+<li>I looked on the server and the Solr cores are there (56GB!), and I don&rsquo;t see any obvious errors in dmesg or anything</li>
+<li>I see that the server hasn&rsquo;t been rebooted in 26 days so I rebooted it</li>
+</ul></li>
+<li>After reboot the Solr stats are still messed up in the Atmire Usage Stats module, it only shows 2019-01!</li>
+</ul>
+
+<p><img src="/cgspace-notes/2019/01/solr-stats-incorrect.png" alt="Solr stats fucked up" /></p>
+
+<ul>
+<li>In the Solr admin UI I see the following error:</li>
+</ul>
+
+<pre><code>statistics-2018: org.apache.solr.common.SolrException:org.apache.solr.common.SolrException: Error opening new searcher
+</code></pre>
+
+<ul>
+<li>Looking in the Solr log I see this:</li>
+</ul>
+
+<pre><code>2019-01-16 13:37:55,395 ERROR org.apache.solr.core.CoreContainer @ Error creating core [statistics-2018]: Error opening new searcher
+org.apache.solr.common.SolrException: Error opening new searcher
+    at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:873)
+    at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:646)
+    at org.apache.solr.core.CoreContainer.create(CoreContainer.java:491)
+    at org.apache.solr.core.CoreContainer.create(CoreContainer.java:466)
+    at org.apache.solr.handler.admin.CoreAdminHandler.handleCreateAction(CoreAdminHandler.java:575)
+    at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestInternal(CoreAdminHandler.java:199)
+    at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:188)
+    at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:135)
+    at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:729)
+    at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:258)
+    at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:207)
+    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
+    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
+    at org.dspace.solr.filters.LocalHostRestrictionFilter.doFilter(LocalHostRestrictionFilter.java:50)
+    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
+    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
+    at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:221)
+    at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:122)
+    at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:505)
+    at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:169)
+    at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103)
+    at org.apache.catalina.valves.CrawlerSessionManagerValve.invoke(CrawlerSessionManagerValve.java:180)
+    at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:956)
+    at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116)
+    at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:436)
+    at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1078)
+    at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:625)
+    at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:316)
+    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
+    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
+    at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
+    at java.lang.Thread.run(Thread.java:748)
+Caused by: org.apache.solr.common.SolrException: Error opening new searcher
+    at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:1565)
+    at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:1677)
+    at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:845)
+    ... 31 more
+Caused by: org.apache.lucene.store.LockObtainFailedException: Lock obtain timed out: NativeFSLock@/home/cgspace.cgiar.org/solr/statistics-2018/data/index/write.lock
+    at org.apache.lucene.store.Lock.obtain(Lock.java:89)
+    at org.apache.lucene.index.IndexWriter.&lt;init&gt;(IndexWriter.java:753)
+    at org.apache.solr.update.SolrIndexWriter.&lt;init&gt;(SolrIndexWriter.java:77)
+    at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:64)
+    at org.apache.solr.update.DefaultSolrCoreState.createMainIndexWriter(DefaultSolrCoreState.java:279)
+    at org.apache.solr.update.DefaultSolrCoreState.getIndexWriter(DefaultSolrCoreState.java:111)
+    at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:1528)
+    ... 33 more
+2019-01-16 13:37:55,401 ERROR org.apache.solr.core.SolrCore @ org.apache.solr.common.SolrException: Error CREATEing SolrCore 'statistics-2018': Unable to create core [statistics-2018] Caused by: Lock obtain timed out: NativeFSLock@/home/cgspace.cgiar.org/solr/statistics-2018/data/index/write.lock
+    at org.apache.solr.handler.admin.CoreAdminHandler.handleCreateAction(CoreAdminHandler.java:613)
+    at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestInternal(CoreAdminHandler.java:199)
+    at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:188)
+    at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:135)
+    at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:729)
+    at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:258)
+    at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:207)
+    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
+    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
+    at org.dspace.solr.filters.LocalHostRestrictionFilter.doFilter(LocalHostRestrictionFilter.java:50)
+    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
+    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
+    at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:221)
+    at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:122)
+    at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:505)
+    at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:169)
+    at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103)
+    at org.apache.catalina.valves.CrawlerSessionManagerValve.invoke(CrawlerSessionManagerValve.java:180)
+    at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:956)
+    at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116)
+    at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:436)
+    at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1078)
+    at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:625)
+    at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:316)
+    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
+    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
+    at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
+    at java.lang.Thread.run(Thread.java:748)
+Caused by: org.apache.solr.common.SolrException: Unable to create core [statistics-2018]
+    at org.apache.solr.core.CoreContainer.create(CoreContainer.java:507)
+    at org.apache.solr.core.CoreContainer.create(CoreContainer.java:466)
+    at org.apache.solr.handler.admin.CoreAdminHandler.handleCreateAction(CoreAdminHandler.java:575)
+    ... 27 more
+Caused by: org.apache.solr.common.SolrException: Error opening new searcher
+    at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:873)
+    at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:646)
+    at org.apache.solr.core.CoreContainer.create(CoreContainer.java:491)
+    ... 29 more
+Caused by: org.apache.solr.common.SolrException: Error opening new searcher
+    at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:1565)
+    at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:1677)
+    at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:845)
+    ... 31 more
+Caused by: org.apache.lucene.store.LockObtainFailedException: Lock obtain timed out: NativeFSLock@/home/cgspace.cgiar.org/solr/statistics-2018/data/index/write.lock
+    at org.apache.lucene.store.Lock.obtain(Lock.java:89)
+    at org.apache.lucene.index.IndexWriter.&lt;init&gt;(IndexWriter.java:753)
+    at org.apache.solr.update.SolrIndexWriter.&lt;init&gt;(SolrIndexWriter.java:77)
+    at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:64)
+    at org.apache.solr.update.DefaultSolrCoreState.createMainIndexWriter(DefaultSolrCoreState.java:279)
+    at org.apache.solr.update.DefaultSolrCoreState.getIndexWriter(DefaultSolrCoreState.java:111)
+    at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:1528)
+    ... 33 more
+</code></pre>
+
+<ul>
+<li>I found some threads on StackOverflow etc discussing this and several suggested increasing the address space for the shell with ulimit</li>
+<li>I added <code>ulimit -v unlimited</code> to the <code>/etc/default/tomcat7</code> and restarted Tomcat and now Solr is working again:</li>
+</ul>
+
+<p><img src="/cgspace-notes/2019/01/solr-stats-incorrect.png" alt="Solr stats working" /></p>
+
 <!-- vim: set sw=2 ts=2: -->
 
   

docs/robots.txt

@@ -43,7 +43,7 @@ Disallow: /cgspace-notes/2015-12/
 Disallow: /cgspace-notes/2015-11/
 Disallow: /cgspace-notes/
 Disallow: /cgspace-notes/categories/
-Disallow: /cgspace-notes/tags/notes/
 Disallow: /cgspace-notes/categories/notes/
+Disallow: /cgspace-notes/tags/notes/
 Disallow: /cgspace-notes/posts/
 Disallow: /cgspace-notes/tags/

docs/sitemap.xml

@@ -4,7 +4,7 @@
   
   <url>
     <loc>https://alanorth.github.io/cgspace-notes/2019-01/</loc>
-    <lastmod>2019-01-15T16:35:16+02:00</lastmod>
+    <lastmod>2019-01-15T16:51:35+02:00</lastmod>
   </url>
   
   <url>
@@ -204,7 +204,7 @@
   
   <url>
     <loc>https://alanorth.github.io/cgspace-notes/</loc>
-    <lastmod>2019-01-15T16:35:16+02:00</lastmod>
+    <lastmod>2019-01-15T16:51:35+02:00</lastmod>
     <priority>0</priority>
   </url>
   
@@ -213,27 +213,27 @@
     <priority>0</priority>
   </url>
   
-  <url>
-    <loc>https://alanorth.github.io/cgspace-notes/tags/notes/</loc>
-    <lastmod>2019-01-15T16:35:16+02:00</lastmod>
-    <priority>0</priority>
-  </url>
-  
   <url>
     <loc>https://alanorth.github.io/cgspace-notes/categories/notes/</loc>
     <lastmod>2018-03-09T22:10:33+02:00</lastmod>
     <priority>0</priority>
   </url>
   
+  <url>
+    <loc>https://alanorth.github.io/cgspace-notes/tags/notes/</loc>
+    <lastmod>2019-01-15T16:51:35+02:00</lastmod>
+    <priority>0</priority>
+  </url>
+  
   <url>
     <loc>https://alanorth.github.io/cgspace-notes/posts/</loc>
-    <lastmod>2019-01-15T16:35:16+02:00</lastmod>
+    <lastmod>2019-01-15T16:51:35+02:00</lastmod>
     <priority>0</priority>
   </url>
   
   <url>
     <loc>https://alanorth.github.io/cgspace-notes/tags/</loc>
-    <lastmod>2019-01-15T16:35:16+02:00</lastmod>
+    <lastmod>2019-01-15T16:51:35+02:00</lastmod>
     <priority>0</priority>
   </url>