From 727c761af45ed2d3693f2ff7c5382d18ab6ca68a Mon Sep 17 00:00:00 2001 From: Alan Orth Date: Wed, 10 Jan 2018 09:39:16 +0200 Subject: [PATCH] Add notes for 2018-01-10 --- content/post/2018-01.md | 92 ++++++++++++++++++++++++++++++++++ public/2018-01/index.html | 103 ++++++++++++++++++++++++++++++++++++-- public/sitemap.xml | 10 ++-- 3 files changed, 197 insertions(+), 8 deletions(-) diff --git a/content/post/2018-01.md b/content/post/2018-01.md index f857c3dcd..2cee8bf46 100644 --- a/content/post/2018-01.md +++ b/content/post/2018-01.md @@ -281,3 +281,95 @@ org.dspace.discovery.SearchServiceException: org.apache.solr.search.SyntaxError: dspace=# \copy (select distinct text_value, count(*) as count from metadatavalue where metadata_field_id = (select metadata_field_id from metadatafieldregistry where element = 'contributor' and qualifier = 'affiliation') AND resource_type_id = 2 group by text_value order by count desc) to /tmp/affiliations.csv with csv; COPY 4515 ``` + +## 2018-01-10 + +- I looked to see what happened to this year's Solr statistics sharding task that should have run on 2018-01-01 and of course it failed: + +``` +Moving: 81742 into core statistics-2010 +Exception: IOException occured when talking to server at: http://localhost:8081/solr//statistics-2010 +org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://localhost:8081/solr//statistics-2010 + at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:566) + at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:210) + at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:206) + at org.dspace.statistics.SolrLogger.shardSolrIndex(SourceFile:2243) + at org.dspace.statistics.util.StatisticsClient.main(StatisticsClient.java:106) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:498) + at org.dspace.app.launcher.ScriptLauncher.runOneCommand(ScriptLauncher.java:226) + at org.dspace.app.launcher.ScriptLauncher.main(ScriptLauncher.java:78) +Caused by: org.apache.http.client.ClientProtocolException + at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:867) + at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82) + at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106) + at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57) + at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:448) + ... 10 more +Caused by: org.apache.http.client.NonRepeatableRequestException: Cannot retry request with a non-repeatable request entity. The cause lists the reason the original request failed. + at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:659) + at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487) + at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863) + ... 14 more +Caused by: java.net.SocketException: Connection reset + at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:115) + at java.net.SocketOutputStream.write(SocketOutputStream.java:155) + at org.apache.http.impl.io.AbstractSessionOutputBuffer.flushBuffer(AbstractSessionOutputBuffer.java:159) + at org.apache.http.impl.io.AbstractSessionOutputBuffer.write(AbstractSessionOutputBuffer.java:179) + at org.apache.http.impl.io.ChunkedOutputStream.flushCacheWithAppend(ChunkedOutputStream.java:124) + at org.apache.http.impl.io.ChunkedOutputStream.write(ChunkedOutputStream.java:181) + at org.apache.http.entity.InputStreamEntity.writeTo(InputStreamEntity.java:132) + at org.apache.http.entity.HttpEntityWrapper.writeTo(HttpEntityWrapper.java:89) + at org.apache.http.impl.client.EntityEnclosingRequestWrapper$EntityWrapper.writeTo(EntityEnclosingRequestWrapper.java:108) + at org.apache.http.impl.entity.EntitySerializer.serialize(EntitySerializer.java:117) + at org.apache.http.impl.AbstractHttpClientConnection.sendRequestEntity(AbstractHttpClientConnection.java:265) + at org.apache.http.impl.conn.ManagedClientConnectionImpl.sendRequestEntity(ManagedClientConnectionImpl.java:203) + at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:236) + at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:121) + at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685) + ... 16 more +``` + +- DSpace Test has the same error but with creating the 2017 core: + +``` +Moving: 2243021 into core statistics-2017 +Exception: IOException occured when talking to server at: http://localhost:8081/solr//statistics-2017 +org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://localhost:8081/solr//statistics-2017 + at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:566) + at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:210) + at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:206) + at org.dspace.statistics.SolrLogger.shardSolrIndex(SourceFile:2243) + at org.dspace.statistics.util.StatisticsClient.main(StatisticsClient.java:106) + at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:498) + at org.dspace.app.launcher.ScriptLauncher.runOneCommand(ScriptLauncher.java:226) + at org.dspace.app.launcher.ScriptLauncher.main(ScriptLauncher.java:78) +Caused by: org.apache.http.client.ClientProtocolException + at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:867) + at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82) + at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106) + at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57) + at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:448) + ... 10 more +``` + +- There is interesting documentation about this on the DSpace Wiki: https://wiki.duraspace.org/display/DSDOC5x/SOLR+Statistics+Maintenance#SOLRStatisticsMaintenance-SolrShardingByYear +- I'm looking to see maybe if we're hitting the issues mentioned in [DS-2212](https://jira.duraspace.org/browse/DS-2212) that were apparently fixed in DSpace 5.2 +- I can apparently search for records in the Solr stats core that have an empty `owningColl` field using this in the Solr admin query: `-owningColl:*` +- On CGSpace I see 48,000,000 records that have an `owningColl` field and 34,000,000 that don't: + +``` +$ http 'http://localhost:3000/solr/statistics/select?q=owningColl%3A*&wt=json&indent=true' | grep numFound + "response":{"numFound":48476327,"start":0,"docs":[ +$ http 'http://localhost:3000/solr/statistics/select?q=-owningColl%3A*&wt=json&indent=true' | grep numFound + "response":{"numFound":34879872,"start":0,"docs":[ +``` + +- Linode continues to test mitigations for Meltdown and Spectre: https://blog.linode.com/2018/01/03/cpu-vulnerabilities-meltdown-spectre/ +- I rebooted DSpace Test to see if the kernel will be updated (currently Linux 4.14.12-x86_64-linode92)... nope. +- It looks like Linode will reboot the KVM hosts later this week, though diff --git a/public/2018-01/index.html b/public/2018-01/index.html index 4a240f61a..709f21fa2 100644 --- a/public/2018-01/index.html +++ b/public/2018-01/index.html @@ -92,7 +92,7 @@ Danny wrote to ask for help renewing the wildcard ilri.org certificate and I adv - + @@ -194,9 +194,9 @@ Danny wrote to ask for help renewing the wildcard ilri.org certificate and I adv "@type": "BlogPosting", "headline": "January, 2018", "url": "https://alanorth.github.io/cgspace-notes/2018-01/", - "wordCount": "1250", + "wordCount": "1602", "datePublished": "2018-01-02T08:35:54-08:00", - "dateModified": "2018-01-09T09:44:16+01:00", + "dateModified": "2018-01-09T10:02:52+01:00", "author": { "@type": "Person", "name": "Alan Orth" @@ -565,6 +565,103 @@ sys 3m14.890s COPY 4515 +

2018-01-10

+ + + +
Moving: 81742 into core statistics-2010
+Exception: IOException occured when talking to server at: http://localhost:8081/solr//statistics-2010
+org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://localhost:8081/solr//statistics-2010
+        at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:566)
+        at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:210)
+        at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:206)
+        at org.dspace.statistics.SolrLogger.shardSolrIndex(SourceFile:2243)
+        at org.dspace.statistics.util.StatisticsClient.main(StatisticsClient.java:106)
+        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
+        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
+        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
+        at java.lang.reflect.Method.invoke(Method.java:498)
+        at org.dspace.app.launcher.ScriptLauncher.runOneCommand(ScriptLauncher.java:226)
+        at org.dspace.app.launcher.ScriptLauncher.main(ScriptLauncher.java:78)
+Caused by: org.apache.http.client.ClientProtocolException
+        at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:867)
+        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
+        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
+        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
+        at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:448)
+        ... 10 more
+Caused by: org.apache.http.client.NonRepeatableRequestException: Cannot retry request with a non-repeatable request entity.  The cause lists the reason the original request failed.
+        at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:659)
+        at org.apache.http.impl.client.DefaultRequestDirector.execute(DefaultRequestDirector.java:487)
+        at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:863)
+        ... 14 more
+Caused by: java.net.SocketException: Connection reset
+        at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:115)
+        at java.net.SocketOutputStream.write(SocketOutputStream.java:155)
+        at org.apache.http.impl.io.AbstractSessionOutputBuffer.flushBuffer(AbstractSessionOutputBuffer.java:159)
+        at org.apache.http.impl.io.AbstractSessionOutputBuffer.write(AbstractSessionOutputBuffer.java:179)
+        at org.apache.http.impl.io.ChunkedOutputStream.flushCacheWithAppend(ChunkedOutputStream.java:124)
+        at org.apache.http.impl.io.ChunkedOutputStream.write(ChunkedOutputStream.java:181)
+        at org.apache.http.entity.InputStreamEntity.writeTo(InputStreamEntity.java:132)
+        at org.apache.http.entity.HttpEntityWrapper.writeTo(HttpEntityWrapper.java:89)
+        at org.apache.http.impl.client.EntityEnclosingRequestWrapper$EntityWrapper.writeTo(EntityEnclosingRequestWrapper.java:108)
+        at org.apache.http.impl.entity.EntitySerializer.serialize(EntitySerializer.java:117)
+        at org.apache.http.impl.AbstractHttpClientConnection.sendRequestEntity(AbstractHttpClientConnection.java:265)
+        at org.apache.http.impl.conn.ManagedClientConnectionImpl.sendRequestEntity(ManagedClientConnectionImpl.java:203)
+        at org.apache.http.protocol.HttpRequestExecutor.doSendRequest(HttpRequestExecutor.java:236)
+        at org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:121)
+        at org.apache.http.impl.client.DefaultRequestDirector.tryExecute(DefaultRequestDirector.java:685)
+        ... 16 more
+
+ + + +
Moving: 2243021 into core statistics-2017
+Exception: IOException occured when talking to server at: http://localhost:8081/solr//statistics-2017
+org.apache.solr.client.solrj.SolrServerException: IOException occured when talking to server at: http://localhost:8081/solr//statistics-2017
+        at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:566)
+        at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:210)
+        at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:206)
+        at org.dspace.statistics.SolrLogger.shardSolrIndex(SourceFile:2243)
+        at org.dspace.statistics.util.StatisticsClient.main(StatisticsClient.java:106)
+        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
+        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
+        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
+        at java.lang.reflect.Method.invoke(Method.java:498)
+        at org.dspace.app.launcher.ScriptLauncher.runOneCommand(ScriptLauncher.java:226)
+        at org.dspace.app.launcher.ScriptLauncher.main(ScriptLauncher.java:78)
+Caused by: org.apache.http.client.ClientProtocolException
+        at org.apache.http.impl.client.AbstractHttpClient.doExecute(AbstractHttpClient.java:867)
+        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
+        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:106)
+        at org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:57)
+        at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:448)
+        ... 10 more
+
+ + + +
$ http 'http://localhost:3000/solr/statistics/select?q=owningColl%3A*&wt=json&indent=true' | grep numFound 
+  "response":{"numFound":48476327,"start":0,"docs":[
+$ http 'http://localhost:3000/solr/statistics/select?q=-owningColl%3A*&wt=json&indent=true' | grep numFound
+  "response":{"numFound":34879872,"start":0,"docs":[
+
+ + + diff --git a/public/sitemap.xml b/public/sitemap.xml index 1d09bb77c..c0577cf39 100644 --- a/public/sitemap.xml +++ b/public/sitemap.xml @@ -4,7 +4,7 @@ https://alanorth.github.io/cgspace-notes/2018-01/ - 2018-01-09T09:44:16+01:00 + 2018-01-09T10:02:52+01:00 @@ -144,7 +144,7 @@ https://alanorth.github.io/cgspace-notes/ - 2018-01-09T09:44:16+01:00 + 2018-01-09T10:02:52+01:00 0 @@ -155,7 +155,7 @@ https://alanorth.github.io/cgspace-notes/tags/notes/ - 2018-01-09T09:44:16+01:00 + 2018-01-09T10:02:52+01:00 0 @@ -167,13 +167,13 @@ https://alanorth.github.io/cgspace-notes/post/ - 2018-01-09T09:44:16+01:00 + 2018-01-09T10:02:52+01:00 0 https://alanorth.github.io/cgspace-notes/tags/ - 2018-01-09T09:44:16+01:00 + 2018-01-09T10:02:52+01:00 0