From ed98e3d8e6d14b3c9ef02b029377707668b85969 Mon Sep 17 00:00:00 2001
From: Alan Orth <alan.orth@gmail.com>
Date: Wed, 16 Jan 2019 16:47:30 +0200
Subject: [PATCH] Add notes for 2019-01-16

---
 content/posts/2019-01.md                | 129 +++++++++++++++++++++
 docs/2019-01/index.html                 | 147 +++++++++++++++++++++++-
 docs/2019/01/solr-stats-correct.png     | Bin 0 -> 3834 bytes
 docs/2019/01/solr-stats-incorrect.png   | Bin 0 -> 3989 bytes
 docs/robots.txt                         |   2 +-
 docs/sitemap.xml                        |  20 ++--
 static/2019/01/solr-stats-correct.png   | Bin 0 -> 3834 bytes
 static/2019/01/solr-stats-incorrect.png | Bin 0 -> 3989 bytes
 8 files changed, 284 insertions(+), 14 deletions(-)
 create mode 100644 docs/2019/01/solr-stats-correct.png
 create mode 100644 docs/2019/01/solr-stats-incorrect.png
 create mode 100644 static/2019/01/solr-stats-correct.png
 create mode 100644 static/2019/01/solr-stats-incorrect.png

diff --git a/content/posts/2019-01.md b/content/posts/2019-01.md
index c6ccbbd6d..d42de8b1c 100644
--- a/content/posts/2019-01.md
+++ b/content/posts/2019-01.md
@@ -358,4 +358,133 @@ $ time http --print h 'https://digitalarchive.worldfishcenter.org/rest/items?exp
    2482 45.5.186.2
 ```
 
+## 2019-01-16
+
+- Day three of CGSpace AReS meeting in Amman
+  - We discussed CGcore and decided some action points
+  - We discussed branding of AReS tool
+- Something happened to the Solr usage statistics on CGSpace
+  - I looked on the server and the Solr cores are there (56GB!), and I don't see any obvious errors in dmesg or anything
+  - I see that the server hasn't been rebooted in 26 days so I rebooted it
+- After reboot the Solr stats are still messed up in the Atmire Usage Stats module, it only shows 2019-01!
+
+![Solr stats fucked up](/cgspace-notes/2019/01/solr-stats-incorrect.png)
+
+- In the Solr admin UI I see the following error:
+
+```
+statistics-2018: org.apache.solr.common.SolrException:org.apache.solr.common.SolrException: Error opening new searcher
+```
+
+- Looking in the Solr log I see this:
+
+```
+2019-01-16 13:37:55,395 ERROR org.apache.solr.core.CoreContainer @ Error creating core [statistics-2018]: Error opening new searcher
+org.apache.solr.common.SolrException: Error opening new searcher
+    at org.apache.solr.core.SolrCore.<init>(SolrCore.java:873)
+    at org.apache.solr.core.SolrCore.<init>(SolrCore.java:646)
+    at org.apache.solr.core.CoreContainer.create(CoreContainer.java:491)
+    at org.apache.solr.core.CoreContainer.create(CoreContainer.java:466)
+    at org.apache.solr.handler.admin.CoreAdminHandler.handleCreateAction(CoreAdminHandler.java:575)
+    at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestInternal(CoreAdminHandler.java:199)
+    at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:188)
+    at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:135)
+    at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:729)
+    at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:258)
+    at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:207)
+    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
+    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
+    at org.dspace.solr.filters.LocalHostRestrictionFilter.doFilter(LocalHostRestrictionFilter.java:50)
+    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
+    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
+    at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:221)
+    at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:122)
+    at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:505)
+    at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:169)
+    at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103)
+    at org.apache.catalina.valves.CrawlerSessionManagerValve.invoke(CrawlerSessionManagerValve.java:180)
+    at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:956)
+    at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116)
+    at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:436)
+    at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1078)
+    at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:625)
+    at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:316)
+    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
+    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
+    at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
+    at java.lang.Thread.run(Thread.java:748)
+Caused by: org.apache.solr.common.SolrException: Error opening new searcher
+    at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:1565)
+    at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:1677)
+    at org.apache.solr.core.SolrCore.<init>(SolrCore.java:845)
+    ... 31 more
+Caused by: org.apache.lucene.store.LockObtainFailedException: Lock obtain timed out: NativeFSLock@/home/cgspace.cgiar.org/solr/statistics-2018/data/index/write.lock
+    at org.apache.lucene.store.Lock.obtain(Lock.java:89)
+    at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:753)
+    at org.apache.solr.update.SolrIndexWriter.<init>(SolrIndexWriter.java:77)
+    at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:64)
+    at org.apache.solr.update.DefaultSolrCoreState.createMainIndexWriter(DefaultSolrCoreState.java:279)
+    at org.apache.solr.update.DefaultSolrCoreState.getIndexWriter(DefaultSolrCoreState.java:111)
+    at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:1528)
+    ... 33 more
+2019-01-16 13:37:55,401 ERROR org.apache.solr.core.SolrCore @ org.apache.solr.common.SolrException: Error CREATEing SolrCore 'statistics-2018': Unable to create core [statistics-2018] Caused by: Lock obtain timed out: NativeFSLock@/home/cgspace.cgiar.org/solr/statistics-2018/data/index/write.lock
+    at org.apache.solr.handler.admin.CoreAdminHandler.handleCreateAction(CoreAdminHandler.java:613)
+    at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestInternal(CoreAdminHandler.java:199)
+    at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:188)
+    at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:135)
+    at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:729)
+    at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:258)
+    at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:207)
+    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
+    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
+    at org.dspace.solr.filters.LocalHostRestrictionFilter.doFilter(LocalHostRestrictionFilter.java:50)
+    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
+    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
+    at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:221)
+    at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:122)
+    at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:505)
+    at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:169)
+    at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103)
+    at org.apache.catalina.valves.CrawlerSessionManagerValve.invoke(CrawlerSessionManagerValve.java:180)
+    at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:956)
+    at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116)
+    at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:436)
+    at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1078)
+    at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:625)
+    at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:316)
+    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
+    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
+    at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
+    at java.lang.Thread.run(Thread.java:748)
+Caused by: org.apache.solr.common.SolrException: Unable to create core [statistics-2018]
+    at org.apache.solr.core.CoreContainer.create(CoreContainer.java:507)
+    at org.apache.solr.core.CoreContainer.create(CoreContainer.java:466)
+    at org.apache.solr.handler.admin.CoreAdminHandler.handleCreateAction(CoreAdminHandler.java:575)
+    ... 27 more
+Caused by: org.apache.solr.common.SolrException: Error opening new searcher
+    at org.apache.solr.core.SolrCore.<init>(SolrCore.java:873)
+    at org.apache.solr.core.SolrCore.<init>(SolrCore.java:646)
+    at org.apache.solr.core.CoreContainer.create(CoreContainer.java:491)
+    ... 29 more
+Caused by: org.apache.solr.common.SolrException: Error opening new searcher
+    at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:1565)
+    at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:1677)
+    at org.apache.solr.core.SolrCore.<init>(SolrCore.java:845)
+    ... 31 more
+Caused by: org.apache.lucene.store.LockObtainFailedException: Lock obtain timed out: NativeFSLock@/home/cgspace.cgiar.org/solr/statistics-2018/data/index/write.lock
+    at org.apache.lucene.store.Lock.obtain(Lock.java:89)
+    at org.apache.lucene.index.IndexWriter.<init>(IndexWriter.java:753)
+    at org.apache.solr.update.SolrIndexWriter.<init>(SolrIndexWriter.java:77)
+    at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:64)
+    at org.apache.solr.update.DefaultSolrCoreState.createMainIndexWriter(DefaultSolrCoreState.java:279)
+    at org.apache.solr.update.DefaultSolrCoreState.getIndexWriter(DefaultSolrCoreState.java:111)
+    at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:1528)
+    ... 33 more
+```
+
+- I found some threads on StackOverflow etc discussing this and several suggested increasing the address space for the shell with ulimit
+- I added `ulimit -v unlimited` to the `/etc/default/tomcat7` and restarted Tomcat and now Solr is working again:
+
+![Solr stats working](/cgspace-notes/2019/01/solr-stats-incorrect.png)
+
 <!-- vim: set sw=2 ts=2: -->
diff --git a/docs/2019-01/index.html b/docs/2019-01/index.html
index 9df14576c..97d9802ab 100644
--- a/docs/2019-01/index.html
+++ b/docs/2019-01/index.html
@@ -27,7 +27,7 @@ I don&rsquo;t see anything interesting in the web server logs around that time t
 " />
 <meta property="og:type" content="article" />
 <meta property="og:url" content="https://alanorth.github.io/cgspace-notes/2019-01/" /><meta property="article:published_time" content="2019-01-02T09:48:30&#43;02:00"/>
-<meta property="article:modified_time" content="2019-01-15T16:35:16&#43;02:00"/>
+<meta property="article:modified_time" content="2019-01-15T16:51:35&#43;02:00"/>
 
 <meta name="twitter:card" content="summary"/>
 <meta name="twitter:title" content="January, 2019"/>
@@ -60,9 +60,9 @@ I don&rsquo;t see anything interesting in the web server logs around that time t
   "@type": "BlogPosting",
   "headline": "January, 2019",
   "url": "https://alanorth.github.io/cgspace-notes/2019-01/",
-  "wordCount": "1581",
+  "wordCount": "2007",
   "datePublished": "2019-01-02T09:48:30&#43;02:00",
-  "dateModified": "2019-01-15T16:35:16&#43;02:00",
+  "dateModified": "2019-01-15T16:51:35&#43;02:00",
   "author": {
     "@type": "Person",
     "name": "Alan Orth"
@@ -529,6 +529,147 @@ In [14]: for row in result.fetchone():
    2482 45.5.186.2
 </code></pre>
 
+<h2 id="2019-01-16">2019-01-16</h2>
+
+<ul>
+<li>Day three of CGSpace AReS meeting in Amman
+
+<ul>
+<li>We discussed CGcore and decided some action points</li>
+<li>We discussed branding of AReS tool</li>
+</ul></li>
+<li>Something happened to the Solr usage statistics on CGSpace
+
+<ul>
+<li>I looked on the server and the Solr cores are there (56GB!), and I don&rsquo;t see any obvious errors in dmesg or anything</li>
+<li>I see that the server hasn&rsquo;t been rebooted in 26 days so I rebooted it</li>
+</ul></li>
+<li>After reboot the Solr stats are still messed up in the Atmire Usage Stats module, it only shows 2019-01!</li>
+</ul>
+
+<p><img src="/cgspace-notes/2019/01/solr-stats-incorrect.png" alt="Solr stats fucked up" /></p>
+
+<ul>
+<li>In the Solr admin UI I see the following error:</li>
+</ul>
+
+<pre><code>statistics-2018: org.apache.solr.common.SolrException:org.apache.solr.common.SolrException: Error opening new searcher
+</code></pre>
+
+<ul>
+<li>Looking in the Solr log I see this:</li>
+</ul>
+
+<pre><code>2019-01-16 13:37:55,395 ERROR org.apache.solr.core.CoreContainer @ Error creating core [statistics-2018]: Error opening new searcher
+org.apache.solr.common.SolrException: Error opening new searcher
+    at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:873)
+    at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:646)
+    at org.apache.solr.core.CoreContainer.create(CoreContainer.java:491)
+    at org.apache.solr.core.CoreContainer.create(CoreContainer.java:466)
+    at org.apache.solr.handler.admin.CoreAdminHandler.handleCreateAction(CoreAdminHandler.java:575)
+    at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestInternal(CoreAdminHandler.java:199)
+    at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:188)
+    at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:135)
+    at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:729)
+    at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:258)
+    at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:207)
+    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
+    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
+    at org.dspace.solr.filters.LocalHostRestrictionFilter.doFilter(LocalHostRestrictionFilter.java:50)
+    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
+    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
+    at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:221)
+    at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:122)
+    at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:505)
+    at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:169)
+    at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103)
+    at org.apache.catalina.valves.CrawlerSessionManagerValve.invoke(CrawlerSessionManagerValve.java:180)
+    at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:956)
+    at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116)
+    at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:436)
+    at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1078)
+    at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:625)
+    at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:316)
+    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
+    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
+    at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
+    at java.lang.Thread.run(Thread.java:748)
+Caused by: org.apache.solr.common.SolrException: Error opening new searcher
+    at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:1565)
+    at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:1677)
+    at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:845)
+    ... 31 more
+Caused by: org.apache.lucene.store.LockObtainFailedException: Lock obtain timed out: NativeFSLock@/home/cgspace.cgiar.org/solr/statistics-2018/data/index/write.lock
+    at org.apache.lucene.store.Lock.obtain(Lock.java:89)
+    at org.apache.lucene.index.IndexWriter.&lt;init&gt;(IndexWriter.java:753)
+    at org.apache.solr.update.SolrIndexWriter.&lt;init&gt;(SolrIndexWriter.java:77)
+    at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:64)
+    at org.apache.solr.update.DefaultSolrCoreState.createMainIndexWriter(DefaultSolrCoreState.java:279)
+    at org.apache.solr.update.DefaultSolrCoreState.getIndexWriter(DefaultSolrCoreState.java:111)
+    at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:1528)
+    ... 33 more
+2019-01-16 13:37:55,401 ERROR org.apache.solr.core.SolrCore @ org.apache.solr.common.SolrException: Error CREATEing SolrCore 'statistics-2018': Unable to create core [statistics-2018] Caused by: Lock obtain timed out: NativeFSLock@/home/cgspace.cgiar.org/solr/statistics-2018/data/index/write.lock
+    at org.apache.solr.handler.admin.CoreAdminHandler.handleCreateAction(CoreAdminHandler.java:613)
+    at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestInternal(CoreAdminHandler.java:199)
+    at org.apache.solr.handler.admin.CoreAdminHandler.handleRequestBody(CoreAdminHandler.java:188)
+    at org.apache.solr.handler.RequestHandlerBase.handleRequest(RequestHandlerBase.java:135)
+    at org.apache.solr.servlet.SolrDispatchFilter.handleAdminRequest(SolrDispatchFilter.java:729)
+    at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:258)
+    at org.apache.solr.servlet.SolrDispatchFilter.doFilter(SolrDispatchFilter.java:207)
+    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
+    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
+    at org.dspace.solr.filters.LocalHostRestrictionFilter.doFilter(LocalHostRestrictionFilter.java:50)
+    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
+    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
+    at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:221)
+    at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:122)
+    at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:505)
+    at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:169)
+    at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103)
+    at org.apache.catalina.valves.CrawlerSessionManagerValve.invoke(CrawlerSessionManagerValve.java:180)
+    at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:956)
+    at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116)
+    at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:436)
+    at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1078)
+    at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:625)
+    at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:316)
+    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
+    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
+    at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
+    at java.lang.Thread.run(Thread.java:748)
+Caused by: org.apache.solr.common.SolrException: Unable to create core [statistics-2018]
+    at org.apache.solr.core.CoreContainer.create(CoreContainer.java:507)
+    at org.apache.solr.core.CoreContainer.create(CoreContainer.java:466)
+    at org.apache.solr.handler.admin.CoreAdminHandler.handleCreateAction(CoreAdminHandler.java:575)
+    ... 27 more
+Caused by: org.apache.solr.common.SolrException: Error opening new searcher
+    at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:873)
+    at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:646)
+    at org.apache.solr.core.CoreContainer.create(CoreContainer.java:491)
+    ... 29 more
+Caused by: org.apache.solr.common.SolrException: Error opening new searcher
+    at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:1565)
+    at org.apache.solr.core.SolrCore.getSearcher(SolrCore.java:1677)
+    at org.apache.solr.core.SolrCore.&lt;init&gt;(SolrCore.java:845)
+    ... 31 more
+Caused by: org.apache.lucene.store.LockObtainFailedException: Lock obtain timed out: NativeFSLock@/home/cgspace.cgiar.org/solr/statistics-2018/data/index/write.lock
+    at org.apache.lucene.store.Lock.obtain(Lock.java:89)
+    at org.apache.lucene.index.IndexWriter.&lt;init&gt;(IndexWriter.java:753)
+    at org.apache.solr.update.SolrIndexWriter.&lt;init&gt;(SolrIndexWriter.java:77)
+    at org.apache.solr.update.SolrIndexWriter.create(SolrIndexWriter.java:64)
+    at org.apache.solr.update.DefaultSolrCoreState.createMainIndexWriter(DefaultSolrCoreState.java:279)
+    at org.apache.solr.update.DefaultSolrCoreState.getIndexWriter(DefaultSolrCoreState.java:111)
+    at org.apache.solr.core.SolrCore.openNewSearcher(SolrCore.java:1528)
+    ... 33 more
+</code></pre>
+
+<ul>
+<li>I found some threads on StackOverflow etc discussing this and several suggested increasing the address space for the shell with ulimit</li>
+<li>I added <code>ulimit -v unlimited</code> to the <code>/etc/default/tomcat7</code> and restarted Tomcat and now Solr is working again:</li>
+</ul>
+
+<p><img src="/cgspace-notes/2019/01/solr-stats-incorrect.png" alt="Solr stats working" /></p>
+
 <!-- vim: set sw=2 ts=2: -->
 
   
diff --git a/docs/2019/01/solr-stats-correct.png b/docs/2019/01/solr-stats-correct.png
new file mode 100644
index 0000000000000000000000000000000000000000..7a170d375c24e4597f0e91d3be061b63ea6fa7ad
GIT binary patch
literal 3834
zcmZu!2{@E%8y=G-TN3FYk>&W5CHpyJ$=FS$;aE~M_6ap*vX?F7C&f7pl5LnKF(-uV
zR4Ou-&R`S~sj)LzhQS#BH+0V5Ip?4Ey5{|!cfR*|-ur&;=XtM5LfTph3rGn-AP`|2
zYjb-D1PVYPTxq*^fh(1T0a~D;iahCHv9<N}%a<=PF)>&ymQMdNI{F@bLDMa5YisNI
z^XGK>1o)AXk}@+ho1C2F;o<rE^{ceBw1tI*ySw|w#>VLA;MUf%hlhutpkQENAPR*N
z78WKD2-(@$5)u+sRaHzTlb@eoQBkq4uTM@+&e74aw6rucGZPAh^6~LCG&ZfRt+lte
zA31VFM@L6YOzg#rmf_*yl9CbzgQ2CR#b&b~Jb0k2tXyATpO%)^)cox8=g%V}Bl7a{
zD=RA)3??!%a_`=~=gyt0uC8utYfVi}jg5^xc<^9nXJ=()rMkNMz`y_u2D7oTDJm+m
zwY80pkH35OE*uUI4h~L9N!i@oTv}T4_xF#8h$t^FpP88{uWo$x`gKxLlBK2P{QNwH
zLWz!!9v>ga<MHpkfAj(QOtLYDJA@6+4hy0N&m3x9=<jB)9<zTU6IxYyAM#qTu&`g0
zq#&_h<I&@$M@=V-=4LmdMY07wtq=2^;N@R4%1@3xIF`7`P<&<8ShRaeZk4GaXL5LZ
zOgWmf1i67tZXb6KC=X!5C4j{6(P+{N={X<EGcrPDi78WJXt(uq>dzvLon!;pjy9K`
z$uKa4>ps>?QolZln`5oOUo>Mi8*uHaUF>;DRZPsjQ-B60?|5iIg{K-jYFu?|{LbT4
zmxGLpUW0Y1p|{_ZUZtfQJ$J*oy>-W%OCjv?SawV-EAweswf<S-2!+r6V-LK~r=~P^
z8<^>Rp-i_evVUu2TqM*+Or2BsE!x<SWvHR?*blKt{q$^v=*{pw){yrfGtIWffJ7y<
zn=$U~O8Qmwo1-q(WxreZSKd~kp%k$0sGrjkS2jh4iYFJkI8n*z*5k|W9nHpDWNVj=
zn~g8gulKu)XnJ=iKmCz5&>9JDdbg^gHy`b(ffC_Gy!tH@-92%l{Y<78^*+NKy>DfH
zY#i{kC9kAaTvbvUJKnsv4$0QVM&{!t*u~i|Op?xE7pZG3$Le8e<L<^^m536w=?51V
zp5%8Yhgue&7E$vhKGaq7#ig&Ov~+w1d~2eR>-~|ZJST3Qe6M|bbC4Fd6x8fzY#IK6
z7tt&A>rzM?Fdu_6=z=K8+q@ZC>wn+BmUC4x0#>K1$NI?beoNlaKsgCd_-*EU`#yFy
zlAJ)8db2L*L}~?3C-zje!s-K2Dvysooo|GeJe+kl5e5>q7(#$3d1F2B^)>2G;qqpP
z$dw#ealX|+a4L#)bwrj>o8J^5HDNbfd9nA@gx$~!gB+`*Gk4x57mp1*scbx+cS7>=
zY_E(Ly<%-HZoU2e;w@MIjZt$U^dFRYk0X`L(V5k<9K!%}f&N@9L%Xu<omSS<0YUmG
z`ZQJrFXY`#o8>zn(xx7GbCDTFy1w|)s1+iBgysb+_X+?e{AAX0NHcc&kQamp;QnsJ
zLlQV+Ss#~4?6hw=|3f%yq)p*e<{U;bFur2LD%{9VgSF_s+#c5J(;#hQ7-}>{y}zjv
zjv9{c3mTZW+;;+yPmkQVT7nlSe%KMBzem_;>+RcT?9Mj#sH$8$^31@8Zoq8y&GU!C
zxZG4)X5vcAWf4nRRu5-0{*@@%0Xr6iO0s_`qI_tNzFT#pD4>~Oe;hf*1b>wxtL+lU
zgD3u0T$~aE3n~UGj^B$HiqI3|8sHNsR$idTD8`ts^OC1MecQvW=%4MIgN8inq=1Du
zt<5S!8jf`9nqTi^xvMbl%zT7X&JpiRW$yitOrb<I+$Y^$b`|nhvS|@*I(jpRu528^
z6ctkHE2Hiqn_`C7a-N15Dfx=~;zDv=yk5E;``s-S4tl<X|A}4{u7ZfXa-M-K<rSd$
z<FB*`e>-PW;-bo^a_5o>7oso@&d&j`L;0VF2N#-RdjMxtV@EeXxOI|DrOr|k3SEW8
z-RkOfKGTiu6Sxt#u)%D9`39>x7@0P04)9!btAqJH>{8Sg&`vIOHNr@Anvtm{lys6+
z_m2|#fb*^dtS<7WPrMv*ID^C6H@0??3@$9h_e^#kVVn}rb-_=<DyIRJAm7-+xz3D$
zY*VxZFxWk7OI61Q^<?oNR0~`~eRE6BS!oN<rhm#HN$0xM^jM-ZBrSTgf(!o0;t>FS
z{15bBp|+!P32?@CrY>|)=b<3T3*`SZU$#DpcKW3><iI_<dv-cB|Ik<v!=Ez;k{AYb
z{GGRNpmrlUgChip#ko?<0O29XU36W&PwIWp0gE=B@z|;)ZR`4q7?;bu0>HoqcO23M
zGt4&g!qdsrPQY%Mv|Q^tTw!$z%qAPlST*to$t(c9*w%7=BhEC&bUVU7cm&M_s?bmZ
zM`kBET0rAqxUuttI^cON_=wfc#{wJ<#}>v_fU*LClc_MkCE(pJo|Yb#6u9O$0l{hh
z(j9ygjv^p4904g0ZHM9%<9Fe<AGewG<R?<Hn!+-3#bA{t6oy^N071|K%oq$HN=6fJ
z-;_q4HEBnk@?M?n?8&0SZSMPNx2LqE6vk<Bbi&ya?7OX#$xdiOx9W~gz)&1v{ylZy
zF^ej7n$89|BHN$YXJ^mBCrEVtAkJAa9Vi|=fg`Cn2jFXSt-%TrAJ$B+Gza;(k9{i0
zk>X$JL$<^(qUGf_z<OwthVH21|Aj{6oA}{K&Ihzlbl^xYyRgqiob(8Kgua54$JI_N
zsW`7<s;$k%lhe~x9j)<`1wv(6IMOxe9NQtAI#`%lE{;Ss#pI&ixm!}YfnQp0a?0sa
z-JnbD1MmSn@~3#|9mQ}^{{u5fKB%Qv`w#@}4;6<_fW)$8_tpH~1{OlHETTh8T38>?
zEU4lv5Vbo%Vi(P{5gie#U<oh{Do5UVP5qtLZTJ6D517vF26lG57N;TeODS*i9z>wV
zB`F*Ul#Bg`qxW=6(o86Dmq*q<U>(r>WpKYgol(&w93xEPf9Nl5!-GI%a3bBLLm-?w
zM)>!rmhIR=%%fM5eDDzB>S>ujk^T!OE;a$Xpl+bN6SM1(+jf58DqYNmR8Y1(@@u4|
z+4EH$)O=`Eo`LMjyWYNBz8cqDonBw9L2ZQch+QWz+zov8ePNgq0%p4@MhmO=q*iq9
zZyEn*-GIpXHDjMWuY-u=<2Pk_cDnkMGB9F3Y=I6rKKa#442a~T0V<(AS#xvI;>h|0
z`_fC~T(CXuQv(bf8)Eih(ILj}Ws6g`FhY=T!(3-_ZaEsq-nipw$^XeQ^-;M&rHP~i
za5&UAmea~i`04-4_rF-!iO=a!JzfBuF2z)tM>=tY=aETL5Ky9^PP;|Jv!c%CA#S>r
z@{&!!Yons(k*Vc!SrC+X4R{$au12PcsvY2COHiWtUQd}1bx{2e*F<Hb-u>{N4*25S
zISQ=jG#AZ;F;;!6&jja+I7B3XG{y8#`1SwRp8r|$e`YU^_q8+F9FS+*cBaH5ibz;;
zk*m?q+CLk}GHdqyh@R8D3y&M0@zRUa)1lwV;%E6U&EIRky{DD?@*5d3=6jf=*i_qb
z@v#(B&otSaV=B&K_^Q+**Hba3oDOOoZMM+tVyW*XUta)^Vtq3G-YTF+NL|jL(C+;Q
zbHQCU`aKf!QI*i3DLVYes4+XOAPlrf91C)seWf65M3s=uvE;s4W)7PpA}L9eH+k0<
zt{zIb(sKLTEd=(soeS!(Z>adb003LVh499|Vv=@!$A-@zcqZz$b%S!Bl%t`+)EB|s
zNM0zJajoD|_4Rqh(L7mHPKf?xh!O%>Izm52SsrS?s2;uUP}h5Fvn&Fb9y4^7eAXEL
zn}&8tVD@#rWM>{?kHO6+cb)vB)bg_{66>;Sn+HDHE)eXuE{_PIq_|s--Qj5y>=Ior
zxLoR^*LCKMm@_PpN7CbdK(?^=%+}UX*oP@MQSoJX)oODiGt4-uRyOpljQ<fgZsLBt
z;>Shfu@fRTiZOK;FJVGS%K9(T0?acM!};XW+(IQFLF%cEX3fhu?5Dd%yHfJ6So-M!
zs)RJ77*aDN*@|SQZL=Dmn02R3w-EYBEQX)tF`X<)#0knC)j51i6c^G7i~$Fz51*FJ
zz$hJis`uw&Z!;myT<+!TX|4xH0&m%~zfwFuEP!v`uy<(NIB0I<<Kl-xi$fZUu4@y-
z4@|sdL0(>LFtKys!UeeKj_4G`={?e^$atVDe7H<C<wD88%#~aAAJ?rawYiB(k_p4P
zrxQk24itMCmU1Io2?#<%o}XonZq$>ef*Hi@4+Lq99r>J|(<dn0PYU?iz^Z*~c6mcD
z?8Q}FYpi`-Z?nQ-U^n@#8$*0AZc6~M^zjZ}4%jVl;|9J@+kqLVd*jCAjFmkfj93vC
U!^m*(FDt~x!q&Xh)cwYP0llxvI{*Lx

literal 0
HcmV?d00001

diff --git a/docs/2019/01/solr-stats-incorrect.png b/docs/2019/01/solr-stats-incorrect.png
new file mode 100644
index 0000000000000000000000000000000000000000..436372d7a064b362a32b6a666ce2cc9c7ba1fad8
GIT binary patch
literal 3989
zcma)9c{r5o8@6Q_Bn+|_VdP}Z5+%DVWvpZ0B}+sh8APTskuWMG8e3$@p6oQXFj<m)
z-}f!XHh$lzbFTASuIqQ+zrO2zzW2GG=f3av`K~Vlc0-E+%n2qVBV*9Mrfx_^Mgb)w
zBY#0d3G5VM+@*kxR+yf##t~_4J$(3ZYHDh1tgp7V7I@+4Xid`k#p>#mkB`sn>?|iI
zr<$7D+}s>BH8mF(mxhMM+S;0(o!#Ev9z8w1y}iA+w>JX=gRQM?T3Xu9&Q5W0F)c0a
zt5>hs+1c~*@?N}naqir??(Xh0XU^p0<Rm604h;<@B_&<Fc=7!C^Cl)Hb#-;p($b$l
ze{N}M+1}pX+}u1oJWNkd2Z2BY0%3A;vaqm_hK43LH<yBfVt9C%l9G~~oLoRapscJ+
zU0t1pg@vD=pGYL`?(TYdd42!>otc?AG&Gcoii(Ymje~=OiHYgNi4#FVL4tyU!otGO
zpFiK<-+%Pz5jQtC7!0<uvKko~(bw0Hjg8I7$hdv`_Sv&%b#--vgM+_*{R)LbArOeF
zs_Ojw{QCO3wYBxu)>c+l*8BJGD=I29H8oG4K0P)z*4Wti^XE?^BcsyNQUn4K9UZ-~
zv2p6usj#rHz`#IJQPKYX{*NC&Iy*aGxNw1$m6eBwM@B{_Dk`eCw|8-Ik&lmWbab?@
zuaA+DQAkKgOiT=i!+CgkeEaq-JUpD2m-o`8OBNOuAt52xu3fu#@1C-<a(jFG_3PKG
ztE*L1R5CL&uUxtE;K75{)m2|#-{s|HZEfxP`ue7(CO8~UM@QG(++0*tR9RDxMx$+P
zY_49tT3%j$aBxsnRTU5rFf%h_Y-}tgCH3;<%gdK9Pft%rL_|bJM!tUiIzK<((a}*+
zQ4t1%#mC1dCnvkRyWhHXOHxvDX=%yd-+z33{MoZ-m6erkZEYeVA`1%(U0q!Y3JNPL
zD-AeYO<jFuRaHerrJtW)TwGjsb~X};l#`RgU@(J&gHN73F*i5I<MFPpuBN7@0|NsI
z2??pGsb9W)>FDUt(a|X=DCp_wk(ZY@G&Fqr^r@w#rJ0!-3Wb`Om~e1#kd>8fXlTG<
zv2Jc|5)u*@Bv$Ik$mkzytE(8_A6OcWxQm|Vsa?;~-f?NCy_SC6k2XA<%X)GKO~tI*
zqJ2$O-f-nkYX(%uVD2Gznz8Z%ISoJNHugqxZw|J!)W$L2Rk12_=SuS1BehY-qWzV^
zrOnnAvytIdf4j|<l_clhL&DNb^61TYz<d1s7<Z8v0t3dK7Az4;0ZAo00qt#^F|rK&
z?B^+2N-S|Q)|%}O%w2T5kxNr@@-pAZkB~rr@69;JH)TwSk46spO!Hn|3^J>p(s<+O
z6j}LCNA8>4iqCi$AIiPUMJWYuHxBDYwFih|m+^!9V#WF8{#BE*eioyXm6}eEd~YU{
zH+b)(7@jAGPPjrZNO=zS-F@XLCyU2(upnHLmt<r^&WFyTP{GDt%Y_=6COkBrqvE4}
zLycjND<<-oXs@^7Q%=VUqaUcMR&3ZJ%C&;IhZC}R_0`PJ5QAq+b_?s}FQ8MBT&Q!c
zU$z*{Bn+ni@a8fyIgsJ6XqR4=f@ha^d3uhDq@^iR;<{lYr#LPT+ql0C+3&>iG~QKf
zo+#>Jvs>vBDRPmMm5tbv_Y{ERYz2Ih0{S7Yk~O0P#Dv_a6?#atiMV2JK9``~o9!JQ
znLG4*k?@l1op7e2jJ6@cTgmgMB0_@B8e%8%V`5h4?;t|7?zyDi&(#TN-rLi4EfJSr
zTWxZ#G3=gB3T&lUnrY}?wle4)_^K9WA~KUmC5kHHA*X<N{iF-%+S_}|%~w8`Yh`@?
z+>CyCNHK4R?*@Ex!?8C(YdS_$Iig(CsrI075D{f~t47nDH8{#)^_#u8^i&K&YTxX3
z$Z|n{uV`m?M#<r>OWV<==u}&Ik;!y#!=5Dv!u39Zi+nLMUwb>)GpPM-3uBWSkS;wq
zz@T&E0f(Vm^B(Q{d^1_a)TOu<MFM{HoefP*o_rcVNdKHxc8%i6<Tgdu<lUshJzx0b
zbiTn-euumI%f2Rqwlt5LLUE;)p<N?xt#vbBVM_&z8@=X(>^9b&M!`9)eGiD(5tw^m
z%^~dNh1@?phl8$_S}0;|Hu@)Os{$CUiE(k0oG}HBa@bpC(QQgQ)5><ey5BpJ{Ua?q
zDbbN0${n`2=wY-*RNe;$-p^TPvgF9vs|%*m5E#_lcA%eI;b6ckCsY-fwtuaDw4=ge
zc!9~qy~(s^^;ze)J*HG!4bP9~f2fGWa&0t5R!Aap9PpZA#^+}R&;|3U=;58$1tu>(
z_-G;ZO_utbk<{o#sAIQE!l;WUYqW{uvxeg&^>5xp@Kq5%FYYEKg!sa8{d)>9FScb+
zB|jr7B?IKbf1Oos(S&YS7&=ku1lw!d2NN!?<OYG!xz+{_aMod@a*G?3Pl9v}mC3>6
z$cL)RXH!qX9s)Ph>Hg@B;C!I+yF1m<f-N8j#c`PT5pzQQXeCslebxue2krla34iC&
z$Hu+jqlm<ZtSuoi#@8DRZr%*VAo;t3LPZYrj*_@hNg{}hFg4a(XfAYM=VvARP;s;f
zB6Tiv%~l*55Ta;AP__rB$d73rgVT^oA(ik85!m}x9SkZ1jS)G>nGbc9PN)Z=JlUKZ
z(+jgt=#5k2aNoKh0O$mT4)KqxMg+Za_jf~90ir2+btsIrfzYebbb>WOx%0-jC0CgQ
zk`a=*7D4Gr^$X$G6#5NK+B<Ha3~3k!v|1+j8i4X+)0k*n0YF<>-$_W*LEy00^)BEP
zR%BNhhLlduh;Sf=NR>e9bKemF_@jYR9gnvE<P>NG&53S?1lNZKX0C-YU-gT<Gxips
z;)|NLJnz@GKVTE@&T)SNQi~u=A*SdL76!~Ghoq5=XoASWR}iTJFhJJ$L#}_53P}6A
z7u8?Xp+7t+bbip%Do5h!WUE6+RERimu2W@xqimO58t8xr5`wd{xk93<;(pX#YXwOg
z3!(3;TVgJNt_s3keFr`=z?t1KEnR>DFlgw1C_?J<SS5hNb4y)`-x75JH8eoZ%RDl~
z{a@gniUS8n&Ntbh3Q#=v%Lcpz5)Y|9sgu9nb2pq7$*KX$=)S)oatsPhQDRmwY=S5O
z^a`K+#B|K#UwZuZ=3n;EVSz}x*#pIjv7wfCNHVa)Osks#61Q#ym#IjSaCmjbS6M~D
zqNv`MHt{b}!z3!zlOJe^0PNR8l)P)B{^t1qq7jk`OH4JcG3kW<>I#lqaKi!B-Gi}R
zf65n2YKj(oIP0*=ow=4!|3gJ&?FWI*%)rdx9*j8dwJCm+#3Zy~R=m0b35fiU>>&i5
zS*?eYJUzTcd#qV`V4ba7Lc$3YF}~@}hyvV*pd*P)`Cdb3ixu}S!Awnk#5ny#`D`P*
zNmU*h;m5E3;S3<_jkAD;ek*i*y<vYb_&0YdSwV&GS5U*Ol+A|)E3GM^7EtG%D?*4{
z6tH60%uYg4H1O#4<6DnYRTdi13-|q^jMon`lqi(+a;5d7|9@?-z+}b2dYh7fLOf`O
z!7w%J<3hs?yYHYA%v0w;1L^w|BquJy(POX^B)PVOuB@JrTXsAqFrOV=S(E(F2{h@Z
z?7k^=jK&eyn#p!-4LsE7HNYVqbhqe0Z(JIg@9T4S8jJW$EsFBfNU#cyoA3wNqr)BC
zVJQE{V(~)xndSWOfF<(eLjXU)tW@j)Q{(|s1Dm4Y3@;6mK|M3qc7gHpPiEX)L)dX1
zlFh7efS0YNU)PH#!xXAJUZr2km~0QynN9x`o!>FyqS_ULtsBWIK8p)N_--p{uO=l0
z<&EzjH8{y_QQl2?b1@I94)2ge>Z7NOEHFj{()Z|xUk6_Hx-(z&2ux`7Xlv%cquY3U
zeU8C-<4`{^1_*io86x&@?|O9i{26qA4g_6)2aEshoEMOim6dpxb^rd>W2s#t3-A~5
z_vP%naTEorm4+1(Y=!?Ej!v9tLW<^;4&(a*_}5n3id_&Dl!JV_e@&b}P<Gr=Si5?&
z9m$wg>Pd#vg=q$lz7wA*h+79O9$nV%|N3k<%gZ}1hxh$bwYTGzO0{Sz|Em_bB70?L
zrg8hh#>TFUPSqiH?~zga@KVz0^1jOC5)hg*@JIqCMbY|$udk12v6Zd$GDssE4-q?D
ziLGq*e_OGc()rcN({p8rJA<tQiw#1=8(K5Sj0VW;^+@{^1Sm2R)w9L(oHW*D(QEFI
z7Py;MExB{jx^Mg04Ae<Y+qQdGPF3kCW%r~c9?4Nt3!lN#r@ITJ2$UJv)t|YW8>oJI
zJ3U$mJ#K(G(<JrOkGiW+R@T(RRN_R(0vF_zgGU!VeHR-&Jsa%7#!0Osif|?g`;V90
zvlKz6BZbiR@i31V7CmKYH?=KgKS?WkW6Y<s@lbmD4yk?`fiXew)1o<tCu_GH{q75*
z6RFd~F>~L>d_2fw!%8g296Ze1+RSS;uW@{oj5ThqUF&Jrcnq?q(mU-yd_4U@lciO{
zHBH%hQTIjZGM9$XYhyUSAewb?%%RpM8t-dYrm0|Iqr_>3PY0PTC8H!Lgy^-o;?HWP
z(K^2uM4z0K*8$;#m>8!zo<C2cFAtX@6QV7V1Ul^uS2fH2e&caxxRfDF>$jpH2+)Xu
rL?TSHu2%i~+{A)`1eltNjI1<H`r#+?stds1G?})>4fQ-#yRd%&8h|O+

literal 0
HcmV?d00001

diff --git a/docs/robots.txt b/docs/robots.txt
index 1405dffd5..c454234ec 100644
--- a/docs/robots.txt
+++ b/docs/robots.txt
@@ -43,7 +43,7 @@ Disallow: /cgspace-notes/2015-12/
 Disallow: /cgspace-notes/2015-11/
 Disallow: /cgspace-notes/
 Disallow: /cgspace-notes/categories/
-Disallow: /cgspace-notes/tags/notes/
 Disallow: /cgspace-notes/categories/notes/
+Disallow: /cgspace-notes/tags/notes/
 Disallow: /cgspace-notes/posts/
 Disallow: /cgspace-notes/tags/
diff --git a/docs/sitemap.xml b/docs/sitemap.xml
index db6821ec9..c2aba4b1b 100644
--- a/docs/sitemap.xml
+++ b/docs/sitemap.xml
@@ -4,7 +4,7 @@
   
   <url>
     <loc>https://alanorth.github.io/cgspace-notes/2019-01/</loc>
-    <lastmod>2019-01-15T16:35:16+02:00</lastmod>
+    <lastmod>2019-01-15T16:51:35+02:00</lastmod>
   </url>
   
   <url>
@@ -204,7 +204,7 @@
   
   <url>
     <loc>https://alanorth.github.io/cgspace-notes/</loc>
-    <lastmod>2019-01-15T16:35:16+02:00</lastmod>
+    <lastmod>2019-01-15T16:51:35+02:00</lastmod>
     <priority>0</priority>
   </url>
   
@@ -213,27 +213,27 @@
     <priority>0</priority>
   </url>
   
-  <url>
-    <loc>https://alanorth.github.io/cgspace-notes/tags/notes/</loc>
-    <lastmod>2019-01-15T16:35:16+02:00</lastmod>
-    <priority>0</priority>
-  </url>
-  
   <url>
     <loc>https://alanorth.github.io/cgspace-notes/categories/notes/</loc>
     <lastmod>2018-03-09T22:10:33+02:00</lastmod>
     <priority>0</priority>
   </url>
   
+  <url>
+    <loc>https://alanorth.github.io/cgspace-notes/tags/notes/</loc>
+    <lastmod>2019-01-15T16:51:35+02:00</lastmod>
+    <priority>0</priority>
+  </url>
+  
   <url>
     <loc>https://alanorth.github.io/cgspace-notes/posts/</loc>
-    <lastmod>2019-01-15T16:35:16+02:00</lastmod>
+    <lastmod>2019-01-15T16:51:35+02:00</lastmod>
     <priority>0</priority>
   </url>
   
   <url>
     <loc>https://alanorth.github.io/cgspace-notes/tags/</loc>
-    <lastmod>2019-01-15T16:35:16+02:00</lastmod>
+    <lastmod>2019-01-15T16:51:35+02:00</lastmod>
     <priority>0</priority>
   </url>
   
diff --git a/static/2019/01/solr-stats-correct.png b/static/2019/01/solr-stats-correct.png
new file mode 100644
index 0000000000000000000000000000000000000000..7a170d375c24e4597f0e91d3be061b63ea6fa7ad
GIT binary patch
literal 3834
zcmZu!2{@E%8y=G-TN3FYk>&W5CHpyJ$=FS$;aE~M_6ap*vX?F7C&f7pl5LnKF(-uV
zR4Ou-&R`S~sj)LzhQS#BH+0V5Ip?4Ey5{|!cfR*|-ur&;=XtM5LfTph3rGn-AP`|2
zYjb-D1PVYPTxq*^fh(1T0a~D;iahCHv9<N}%a<=PF)>&ymQMdNI{F@bLDMa5YisNI
z^XGK>1o)AXk}@+ho1C2F;o<rE^{ceBw1tI*ySw|w#>VLA;MUf%hlhutpkQENAPR*N
z78WKD2-(@$5)u+sRaHzTlb@eoQBkq4uTM@+&e74aw6rucGZPAh^6~LCG&ZfRt+lte
zA31VFM@L6YOzg#rmf_*yl9CbzgQ2CR#b&b~Jb0k2tXyATpO%)^)cox8=g%V}Bl7a{
zD=RA)3??!%a_`=~=gyt0uC8utYfVi}jg5^xc<^9nXJ=()rMkNMz`y_u2D7oTDJm+m
zwY80pkH35OE*uUI4h~L9N!i@oTv}T4_xF#8h$t^FpP88{uWo$x`gKxLlBK2P{QNwH
zLWz!!9v>ga<MHpkfAj(QOtLYDJA@6+4hy0N&m3x9=<jB)9<zTU6IxYyAM#qTu&`g0
zq#&_h<I&@$M@=V-=4LmdMY07wtq=2^;N@R4%1@3xIF`7`P<&<8ShRaeZk4GaXL5LZ
zOgWmf1i67tZXb6KC=X!5C4j{6(P+{N={X<EGcrPDi78WJXt(uq>dzvLon!;pjy9K`
z$uKa4>ps>?QolZln`5oOUo>Mi8*uHaUF>;DRZPsjQ-B60?|5iIg{K-jYFu?|{LbT4
zmxGLpUW0Y1p|{_ZUZtfQJ$J*oy>-W%OCjv?SawV-EAweswf<S-2!+r6V-LK~r=~P^
z8<^>Rp-i_evVUu2TqM*+Or2BsE!x<SWvHR?*blKt{q$^v=*{pw){yrfGtIWffJ7y<
zn=$U~O8Qmwo1-q(WxreZSKd~kp%k$0sGrjkS2jh4iYFJkI8n*z*5k|W9nHpDWNVj=
zn~g8gulKu)XnJ=iKmCz5&>9JDdbg^gHy`b(ffC_Gy!tH@-92%l{Y<78^*+NKy>DfH
zY#i{kC9kAaTvbvUJKnsv4$0QVM&{!t*u~i|Op?xE7pZG3$Le8e<L<^^m536w=?51V
zp5%8Yhgue&7E$vhKGaq7#ig&Ov~+w1d~2eR>-~|ZJST3Qe6M|bbC4Fd6x8fzY#IK6
z7tt&A>rzM?Fdu_6=z=K8+q@ZC>wn+BmUC4x0#>K1$NI?beoNlaKsgCd_-*EU`#yFy
zlAJ)8db2L*L}~?3C-zje!s-K2Dvysooo|GeJe+kl5e5>q7(#$3d1F2B^)>2G;qqpP
z$dw#ealX|+a4L#)bwrj>o8J^5HDNbfd9nA@gx$~!gB+`*Gk4x57mp1*scbx+cS7>=
zY_E(Ly<%-HZoU2e;w@MIjZt$U^dFRYk0X`L(V5k<9K!%}f&N@9L%Xu<omSS<0YUmG
z`ZQJrFXY`#o8>zn(xx7GbCDTFy1w|)s1+iBgysb+_X+?e{AAX0NHcc&kQamp;QnsJ
zLlQV+Ss#~4?6hw=|3f%yq)p*e<{U;bFur2LD%{9VgSF_s+#c5J(;#hQ7-}>{y}zjv
zjv9{c3mTZW+;;+yPmkQVT7nlSe%KMBzem_;>+RcT?9Mj#sH$8$^31@8Zoq8y&GU!C
zxZG4)X5vcAWf4nRRu5-0{*@@%0Xr6iO0s_`qI_tNzFT#pD4>~Oe;hf*1b>wxtL+lU
zgD3u0T$~aE3n~UGj^B$HiqI3|8sHNsR$idTD8`ts^OC1MecQvW=%4MIgN8inq=1Du
zt<5S!8jf`9nqTi^xvMbl%zT7X&JpiRW$yitOrb<I+$Y^$b`|nhvS|@*I(jpRu528^
z6ctkHE2Hiqn_`C7a-N15Dfx=~;zDv=yk5E;``s-S4tl<X|A}4{u7ZfXa-M-K<rSd$
z<FB*`e>-PW;-bo^a_5o>7oso@&d&j`L;0VF2N#-RdjMxtV@EeXxOI|DrOr|k3SEW8
z-RkOfKGTiu6Sxt#u)%D9`39>x7@0P04)9!btAqJH>{8Sg&`vIOHNr@Anvtm{lys6+
z_m2|#fb*^dtS<7WPrMv*ID^C6H@0??3@$9h_e^#kVVn}rb-_=<DyIRJAm7-+xz3D$
zY*VxZFxWk7OI61Q^<?oNR0~`~eRE6BS!oN<rhm#HN$0xM^jM-ZBrSTgf(!o0;t>FS
z{15bBp|+!P32?@CrY>|)=b<3T3*`SZU$#DpcKW3><iI_<dv-cB|Ik<v!=Ez;k{AYb
z{GGRNpmrlUgChip#ko?<0O29XU36W&PwIWp0gE=B@z|;)ZR`4q7?;bu0>HoqcO23M
zGt4&g!qdsrPQY%Mv|Q^tTw!$z%qAPlST*to$t(c9*w%7=BhEC&bUVU7cm&M_s?bmZ
zM`kBET0rAqxUuttI^cON_=wfc#{wJ<#}>v_fU*LClc_MkCE(pJo|Yb#6u9O$0l{hh
z(j9ygjv^p4904g0ZHM9%<9Fe<AGewG<R?<Hn!+-3#bA{t6oy^N071|K%oq$HN=6fJ
z-;_q4HEBnk@?M?n?8&0SZSMPNx2LqE6vk<Bbi&ya?7OX#$xdiOx9W~gz)&1v{ylZy
zF^ej7n$89|BHN$YXJ^mBCrEVtAkJAa9Vi|=fg`Cn2jFXSt-%TrAJ$B+Gza;(k9{i0
zk>X$JL$<^(qUGf_z<OwthVH21|Aj{6oA}{K&Ihzlbl^xYyRgqiob(8Kgua54$JI_N
zsW`7<s;$k%lhe~x9j)<`1wv(6IMOxe9NQtAI#`%lE{;Ss#pI&ixm!}YfnQp0a?0sa
z-JnbD1MmSn@~3#|9mQ}^{{u5fKB%Qv`w#@}4;6<_fW)$8_tpH~1{OlHETTh8T38>?
zEU4lv5Vbo%Vi(P{5gie#U<oh{Do5UVP5qtLZTJ6D517vF26lG57N;TeODS*i9z>wV
zB`F*Ul#Bg`qxW=6(o86Dmq*q<U>(r>WpKYgol(&w93xEPf9Nl5!-GI%a3bBLLm-?w
zM)>!rmhIR=%%fM5eDDzB>S>ujk^T!OE;a$Xpl+bN6SM1(+jf58DqYNmR8Y1(@@u4|
z+4EH$)O=`Eo`LMjyWYNBz8cqDonBw9L2ZQch+QWz+zov8ePNgq0%p4@MhmO=q*iq9
zZyEn*-GIpXHDjMWuY-u=<2Pk_cDnkMGB9F3Y=I6rKKa#442a~T0V<(AS#xvI;>h|0
z`_fC~T(CXuQv(bf8)Eih(ILj}Ws6g`FhY=T!(3-_ZaEsq-nipw$^XeQ^-;M&rHP~i
za5&UAmea~i`04-4_rF-!iO=a!JzfBuF2z)tM>=tY=aETL5Ky9^PP;|Jv!c%CA#S>r
z@{&!!Yons(k*Vc!SrC+X4R{$au12PcsvY2COHiWtUQd}1bx{2e*F<Hb-u>{N4*25S
zISQ=jG#AZ;F;;!6&jja+I7B3XG{y8#`1SwRp8r|$e`YU^_q8+F9FS+*cBaH5ibz;;
zk*m?q+CLk}GHdqyh@R8D3y&M0@zRUa)1lwV;%E6U&EIRky{DD?@*5d3=6jf=*i_qb
z@v#(B&otSaV=B&K_^Q+**Hba3oDOOoZMM+tVyW*XUta)^Vtq3G-YTF+NL|jL(C+;Q
zbHQCU`aKf!QI*i3DLVYes4+XOAPlrf91C)seWf65M3s=uvE;s4W)7PpA}L9eH+k0<
zt{zIb(sKLTEd=(soeS!(Z>adb003LVh499|Vv=@!$A-@zcqZz$b%S!Bl%t`+)EB|s
zNM0zJajoD|_4Rqh(L7mHPKf?xh!O%>Izm52SsrS?s2;uUP}h5Fvn&Fb9y4^7eAXEL
zn}&8tVD@#rWM>{?kHO6+cb)vB)bg_{66>;Sn+HDHE)eXuE{_PIq_|s--Qj5y>=Ior
zxLoR^*LCKMm@_PpN7CbdK(?^=%+}UX*oP@MQSoJX)oODiGt4-uRyOpljQ<fgZsLBt
z;>Shfu@fRTiZOK;FJVGS%K9(T0?acM!};XW+(IQFLF%cEX3fhu?5Dd%yHfJ6So-M!
zs)RJ77*aDN*@|SQZL=Dmn02R3w-EYBEQX)tF`X<)#0knC)j51i6c^G7i~$Fz51*FJ
zz$hJis`uw&Z!;myT<+!TX|4xH0&m%~zfwFuEP!v`uy<(NIB0I<<Kl-xi$fZUu4@y-
z4@|sdL0(>LFtKys!UeeKj_4G`={?e^$atVDe7H<C<wD88%#~aAAJ?rawYiB(k_p4P
zrxQk24itMCmU1Io2?#<%o}XonZq$>ef*Hi@4+Lq99r>J|(<dn0PYU?iz^Z*~c6mcD
z?8Q}FYpi`-Z?nQ-U^n@#8$*0AZc6~M^zjZ}4%jVl;|9J@+kqLVd*jCAjFmkfj93vC
U!^m*(FDt~x!q&Xh)cwYP0llxvI{*Lx

literal 0
HcmV?d00001

diff --git a/static/2019/01/solr-stats-incorrect.png b/static/2019/01/solr-stats-incorrect.png
new file mode 100644
index 0000000000000000000000000000000000000000..436372d7a064b362a32b6a666ce2cc9c7ba1fad8
GIT binary patch
literal 3989
zcma)9c{r5o8@6Q_Bn+|_VdP}Z5+%DVWvpZ0B}+sh8APTskuWMG8e3$@p6oQXFj<m)
z-}f!XHh$lzbFTASuIqQ+zrO2zzW2GG=f3av`K~Vlc0-E+%n2qVBV*9Mrfx_^Mgb)w
zBY#0d3G5VM+@*kxR+yf##t~_4J$(3ZYHDh1tgp7V7I@+4Xid`k#p>#mkB`sn>?|iI
zr<$7D+}s>BH8mF(mxhMM+S;0(o!#Ev9z8w1y}iA+w>JX=gRQM?T3Xu9&Q5W0F)c0a
zt5>hs+1c~*@?N}naqir??(Xh0XU^p0<Rm604h;<@B_&<Fc=7!C^Cl)Hb#-;p($b$l
ze{N}M+1}pX+}u1oJWNkd2Z2BY0%3A;vaqm_hK43LH<yBfVt9C%l9G~~oLoRapscJ+
zU0t1pg@vD=pGYL`?(TYdd42!>otc?AG&Gcoii(Ymje~=OiHYgNi4#FVL4tyU!otGO
zpFiK<-+%Pz5jQtC7!0<uvKko~(bw0Hjg8I7$hdv`_Sv&%b#--vgM+_*{R)LbArOeF
zs_Ojw{QCO3wYBxu)>c+l*8BJGD=I29H8oG4K0P)z*4Wti^XE?^BcsyNQUn4K9UZ-~
zv2p6usj#rHz`#IJQPKYX{*NC&Iy*aGxNw1$m6eBwM@B{_Dk`eCw|8-Ik&lmWbab?@
zuaA+DQAkKgOiT=i!+CgkeEaq-JUpD2m-o`8OBNOuAt52xu3fu#@1C-<a(jFG_3PKG
ztE*L1R5CL&uUxtE;K75{)m2|#-{s|HZEfxP`ue7(CO8~UM@QG(++0*tR9RDxMx$+P
zY_49tT3%j$aBxsnRTU5rFf%h_Y-}tgCH3;<%gdK9Pft%rL_|bJM!tUiIzK<((a}*+
zQ4t1%#mC1dCnvkRyWhHXOHxvDX=%yd-+z33{MoZ-m6erkZEYeVA`1%(U0q!Y3JNPL
zD-AeYO<jFuRaHerrJtW)TwGjsb~X};l#`RgU@(J&gHN73F*i5I<MFPpuBN7@0|NsI
z2??pGsb9W)>FDUt(a|X=DCp_wk(ZY@G&Fqr^r@w#rJ0!-3Wb`Om~e1#kd>8fXlTG<
zv2Jc|5)u*@Bv$Ik$mkzytE(8_A6OcWxQm|Vsa?;~-f?NCy_SC6k2XA<%X)GKO~tI*
zqJ2$O-f-nkYX(%uVD2Gznz8Z%ISoJNHugqxZw|J!)W$L2Rk12_=SuS1BehY-qWzV^
zrOnnAvytIdf4j|<l_clhL&DNb^61TYz<d1s7<Z8v0t3dK7Az4;0ZAo00qt#^F|rK&
z?B^+2N-S|Q)|%}O%w2T5kxNr@@-pAZkB~rr@69;JH)TwSk46spO!Hn|3^J>p(s<+O
z6j}LCNA8>4iqCi$AIiPUMJWYuHxBDYwFih|m+^!9V#WF8{#BE*eioyXm6}eEd~YU{
zH+b)(7@jAGPPjrZNO=zS-F@XLCyU2(upnHLmt<r^&WFyTP{GDt%Y_=6COkBrqvE4}
zLycjND<<-oXs@^7Q%=VUqaUcMR&3ZJ%C&;IhZC}R_0`PJ5QAq+b_?s}FQ8MBT&Q!c
zU$z*{Bn+ni@a8fyIgsJ6XqR4=f@ha^d3uhDq@^iR;<{lYr#LPT+ql0C+3&>iG~QKf
zo+#>Jvs>vBDRPmMm5tbv_Y{ERYz2Ih0{S7Yk~O0P#Dv_a6?#atiMV2JK9``~o9!JQ
znLG4*k?@l1op7e2jJ6@cTgmgMB0_@B8e%8%V`5h4?;t|7?zyDi&(#TN-rLi4EfJSr
zTWxZ#G3=gB3T&lUnrY}?wle4)_^K9WA~KUmC5kHHA*X<N{iF-%+S_}|%~w8`Yh`@?
z+>CyCNHK4R?*@Ex!?8C(YdS_$Iig(CsrI075D{f~t47nDH8{#)^_#u8^i&K&YTxX3
z$Z|n{uV`m?M#<r>OWV<==u}&Ik;!y#!=5Dv!u39Zi+nLMUwb>)GpPM-3uBWSkS;wq
zz@T&E0f(Vm^B(Q{d^1_a)TOu<MFM{HoefP*o_rcVNdKHxc8%i6<Tgdu<lUshJzx0b
zbiTn-euumI%f2Rqwlt5LLUE;)p<N?xt#vbBVM_&z8@=X(>^9b&M!`9)eGiD(5tw^m
z%^~dNh1@?phl8$_S}0;|Hu@)Os{$CUiE(k0oG}HBa@bpC(QQgQ)5><ey5BpJ{Ua?q
zDbbN0${n`2=wY-*RNe;$-p^TPvgF9vs|%*m5E#_lcA%eI;b6ckCsY-fwtuaDw4=ge
zc!9~qy~(s^^;ze)J*HG!4bP9~f2fGWa&0t5R!Aap9PpZA#^+}R&;|3U=;58$1tu>(
z_-G;ZO_utbk<{o#sAIQE!l;WUYqW{uvxeg&^>5xp@Kq5%FYYEKg!sa8{d)>9FScb+
zB|jr7B?IKbf1Oos(S&YS7&=ku1lw!d2NN!?<OYG!xz+{_aMod@a*G?3Pl9v}mC3>6
z$cL)RXH!qX9s)Ph>Hg@B;C!I+yF1m<f-N8j#c`PT5pzQQXeCslebxue2krla34iC&
z$Hu+jqlm<ZtSuoi#@8DRZr%*VAo;t3LPZYrj*_@hNg{}hFg4a(XfAYM=VvARP;s;f
zB6Tiv%~l*55Ta;AP__rB$d73rgVT^oA(ik85!m}x9SkZ1jS)G>nGbc9PN)Z=JlUKZ
z(+jgt=#5k2aNoKh0O$mT4)KqxMg+Za_jf~90ir2+btsIrfzYebbb>WOx%0-jC0CgQ
zk`a=*7D4Gr^$X$G6#5NK+B<Ha3~3k!v|1+j8i4X+)0k*n0YF<>-$_W*LEy00^)BEP
zR%BNhhLlduh;Sf=NR>e9bKemF_@jYR9gnvE<P>NG&53S?1lNZKX0C-YU-gT<Gxips
z;)|NLJnz@GKVTE@&T)SNQi~u=A*SdL76!~Ghoq5=XoASWR}iTJFhJJ$L#}_53P}6A
z7u8?Xp+7t+bbip%Do5h!WUE6+RERimu2W@xqimO58t8xr5`wd{xk93<;(pX#YXwOg
z3!(3;TVgJNt_s3keFr`=z?t1KEnR>DFlgw1C_?J<SS5hNb4y)`-x75JH8eoZ%RDl~
z{a@gniUS8n&Ntbh3Q#=v%Lcpz5)Y|9sgu9nb2pq7$*KX$=)S)oatsPhQDRmwY=S5O
z^a`K+#B|K#UwZuZ=3n;EVSz}x*#pIjv7wfCNHVa)Osks#61Q#ym#IjSaCmjbS6M~D
zqNv`MHt{b}!z3!zlOJe^0PNR8l)P)B{^t1qq7jk`OH4JcG3kW<>I#lqaKi!B-Gi}R
zf65n2YKj(oIP0*=ow=4!|3gJ&?FWI*%)rdx9*j8dwJCm+#3Zy~R=m0b35fiU>>&i5
zS*?eYJUzTcd#qV`V4ba7Lc$3YF}~@}hyvV*pd*P)`Cdb3ixu}S!Awnk#5ny#`D`P*
zNmU*h;m5E3;S3<_jkAD;ek*i*y<vYb_&0YdSwV&GS5U*Ol+A|)E3GM^7EtG%D?*4{
z6tH60%uYg4H1O#4<6DnYRTdi13-|q^jMon`lqi(+a;5d7|9@?-z+}b2dYh7fLOf`O
z!7w%J<3hs?yYHYA%v0w;1L^w|BquJy(POX^B)PVOuB@JrTXsAqFrOV=S(E(F2{h@Z
z?7k^=jK&eyn#p!-4LsE7HNYVqbhqe0Z(JIg@9T4S8jJW$EsFBfNU#cyoA3wNqr)BC
zVJQE{V(~)xndSWOfF<(eLjXU)tW@j)Q{(|s1Dm4Y3@;6mK|M3qc7gHpPiEX)L)dX1
zlFh7efS0YNU)PH#!xXAJUZr2km~0QynN9x`o!>FyqS_ULtsBWIK8p)N_--p{uO=l0
z<&EzjH8{y_QQl2?b1@I94)2ge>Z7NOEHFj{()Z|xUk6_Hx-(z&2ux`7Xlv%cquY3U
zeU8C-<4`{^1_*io86x&@?|O9i{26qA4g_6)2aEshoEMOim6dpxb^rd>W2s#t3-A~5
z_vP%naTEorm4+1(Y=!?Ej!v9tLW<^;4&(a*_}5n3id_&Dl!JV_e@&b}P<Gr=Si5?&
z9m$wg>Pd#vg=q$lz7wA*h+79O9$nV%|N3k<%gZ}1hxh$bwYTGzO0{Sz|Em_bB70?L
zrg8hh#>TFUPSqiH?~zga@KVz0^1jOC5)hg*@JIqCMbY|$udk12v6Zd$GDssE4-q?D
ziLGq*e_OGc()rcN({p8rJA<tQiw#1=8(K5Sj0VW;^+@{^1Sm2R)w9L(oHW*D(QEFI
z7Py;MExB{jx^Mg04Ae<Y+qQdGPF3kCW%r~c9?4Nt3!lN#r@ITJ2$UJv)t|YW8>oJI
zJ3U$mJ#K(G(<JrOkGiW+R@T(RRN_R(0vF_zgGU!VeHR-&Jsa%7#!0Osif|?g`;V90
zvlKz6BZbiR@i31V7CmKYH?=KgKS?WkW6Y<s@lbmD4yk?`fiXew)1o<tCu_GH{q75*
z6RFd~F>~L>d_2fw!%8g296Ze1+RSS;uW@{oj5ThqUF&Jrcnq?q(mU-yd_4U@lciO{
zHBH%hQTIjZGM9$XYhyUSAewb?%%RpM8t-dYrm0|Iqr_>3PY0PTC8H!Lgy^-o;?HWP
z(K^2uM4z0K*8$;#m>8!zo<C2cFAtX@6QV7V1Ul^uS2fH2e&caxxRfDF>$jpH2+)Xu
rL?TSHu2%i~+{A)`1eltNjI1<H`r#+?stds1G?})>4fQ-#yRd%&8h|O+

literal 0
HcmV?d00001