From 823c8ddab4ec4f377bedce6b951cb7dd368758de Mon Sep 17 00:00:00 2001 From: Alan Orth Date: Fri, 2 Dec 2016 13:13:58 +0200 Subject: [PATCH] Add notes for 2016-12-02 --- content/post/2016-11.md | 1 + content/post/2016-12.md | 130 ++++++++++++ public/2015-11/index.html | 4 +- public/2015-12/index.html | 4 +- public/2016-01/index.html | 4 +- public/2016-02/index.html | 4 +- public/2016-03/index.html | 4 +- public/2016-04/index.html | 4 +- public/2016-05/index.html | 4 +- public/2016-06/index.html | 4 +- public/2016-07/index.html | 4 +- public/2016-08/index.html | 4 +- public/2016-09/index.html | 4 +- public/2016-10/index.html | 4 +- public/2016-11/index.html | 7 +- public/2016-12/index.html | 312 ++++++++++++++++++++++++++++ public/index.html | 53 ++--- public/index.xml | 143 ++++++++++++- public/page/2/index.html | 37 +++- public/post/index.html | 53 ++--- public/post/index.xml | 143 ++++++++++++- public/post/page/2/index.html | 37 +++- public/robots.txt | 1 + public/sitemap.xml | 7 +- public/tags/notes/index.html | 53 ++--- public/tags/notes/index.xml | 141 +++++++++++++ public/tags/notes/page/2/index.html | 37 +++- 27 files changed, 1062 insertions(+), 141 deletions(-) create mode 100644 content/post/2016-12.md create mode 100644 public/2016-12/index.html diff --git a/content/post/2016-11.md b/content/post/2016-11.md index d416190ba..eff8bd96b 100644 --- a/content/post/2016-11.md +++ b/content/post/2016-11.md @@ -475,3 +475,4 @@ org.dspace.discovery.SearchServiceException: Error executing query - The `maxHttpHeaderSize` fix worked on CGSpace (user is able to see the community list on the homepage) - The "take task" cache fix worked on DSpace Test but it's not an official patch, so I'll have to report the bug to DSpace people and try to get advice +- More work on the KM4Dev Journal article diff --git a/content/post/2016-12.md b/content/post/2016-12.md new file mode 100644 index 000000000..c13ad9ebf --- /dev/null +++ b/content/post/2016-12.md @@ -0,0 +1,130 @@ ++++ +date = "2016-12-02T10:43:00+03:00" +author = "Alan Orth" +title = "December, 2016" +tags = ["Notes"] + ++++ +## 2016-12-02 + +- CGSpace was down for five hours in the morning while I was sleeping +- While looking in the logs for errors, I see tons of warnings about Atmire MQM: + +``` +2016-12-02 03:00:32,352 WARN com.atmire.metadataquality.batchedit.BatchEditConsumer @ BatchEditConsumer should not have been given this kind of Subject in an event, skipping: org.dspace.event.Event(eventType=CREATE, SubjectType=BUNDLE, SubjectID=70316, ObjectType=(Unknown), ObjectID=-1, TimeStamp=1480647632305, dispatcher=1544803905, detail=[null], transactionID="TX157907838689377964651674089851855413607") +2016-12-02 03:00:32,353 WARN com.atmire.metadataquality.batchedit.BatchEditConsumer @ BatchEditConsumer should not have been given this kind of Subject in an event, skipping: org.dspace.event.Event(eventType=MODIFY_METADATA, SubjectType=BUNDLE, SubjectID =70316, ObjectType=(Unknown), ObjectID=-1, TimeStamp=1480647632309, dispatcher=1544803905, detail="dc.title", transactionID="TX157907838689377964651674089851855413607") +2016-12-02 03:00:32,353 WARN com.atmire.metadataquality.batchedit.BatchEditConsumer @ BatchEditConsumer should not have been given this kind of Subject in an event, skipping: org.dspace.event.Event(eventType=ADD, SubjectType=ITEM, SubjectID=80044, Object Type=BUNDLE, ObjectID=70316, TimeStamp=1480647632311, dispatcher=1544803905, detail="THUMBNAIL", transactionID="TX157907838689377964651674089851855413607") +2016-12-02 03:00:32,353 WARN com.atmire.metadataquality.batchedit.BatchEditConsumer @ BatchEditConsumer should not have been given this kind of Subject in an event, skipping: org.dspace.event.Event(eventType=ADD, SubjectType=BUNDLE, SubjectID=70316, Obje ctType=BITSTREAM, ObjectID=86715, TimeStamp=1480647632318, dispatcher=1544803905, detail="-1", transactionID="TX157907838689377964651674089851855413607") +2016-12-02 03:00:32,353 WARN com.atmire.metadataquality.batchedit.BatchEditConsumer @ BatchEditConsumer should not have been given this kind of Subject in an event, skipping: org.dspace.event.Event(eventType=MODIFY, SubjectType=ITEM, SubjectID=80044, ObjectType=(Unknown), ObjectID=-1, TimeStamp=1480647632351, dispatcher=1544803905, detail=[null], transactionID="TX157907838689377964651674089851855413607") +``` + +- I see thousands of them in the logs for the last few months, so it's not related to the DSpace 5.5 upgrade +- I've raised a ticket with Atmire to ask +- Another worrying error from dspace.log is: + +``` +org.springframework.web.util.NestedServletException: Handler processing failed; nested exception is java.lang.NoSuchMethodError: com.atmire.statistics.generator.DSpaceObjectDatasetGenerator.toDatasetQuery(Lorg/dspace/core/Context;)Lcom/atmire/statistics/content/DatasetQuery; + at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:972) + at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:852) + at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:882) + at org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:789) + at javax.servlet.http.HttpServlet.service(HttpServlet.java:646) + at javax.servlet.http.HttpServlet.service(HttpServlet.java:727) + at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:303) + at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) + at org.dspace.app.xmlui.cocoon.SetCharacterEncodingFilter.doFilter(SetCharacterEncodingFilter.java:111) + at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) + at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) + at org.dspace.app.xmlui.cocoon.DSpaceCocoonServletFilter.doFilter(DSpaceCocoonServletFilter.java:274) + at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) + at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) + at org.dspace.app.xmlui.cocoon.servlet.multipart.DSpaceMultipartFilter.doFilter(DSpaceMultipartFilter.java:119) + at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) + at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) + at org.dspace.utils.servlet.DSpaceWebappServletFilter.doFilter(DSpaceWebappServletFilter.java:78) + at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) + at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) + at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:220) + at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:122) + at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:501) + at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:170) + at com.googlecode.psiprobe.Tomcat70AgentValve.invoke(Tomcat70AgentValve.java:44) + at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:98) + at org.apache.catalina.valves.CrawlerSessionManagerValve.invoke(CrawlerSessionManagerValve.java:180) + at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:950) + at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116) + at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:408) + at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1041) + at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:607) + at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:313) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) + at java.lang.Thread.run(Thread.java:745) +Caused by: java.lang.NoSuchMethodError: com.atmire.statistics.generator.DSpaceObjectDatasetGenerator.toDatasetQuery(Lorg/dspace/core/Context;)Lcom/atmire/statistics/content/DatasetQuery; + at com.atmire.statistics.generator.TopNDSODatasetGenerator.toDatasetQuery(SourceFile:39) + at com.atmire.statistics.display.StatisticsDataVisitsMultidata.createDataset(SourceFile:108) + at org.dspace.statistics.content.StatisticsDisplay.createDataset(SourceFile:384) + at org.dspace.statistics.content.StatisticsDisplay.getDataset(SourceFile:404) + at com.atmire.statistics.mostpopular.JSONStatsMostPopularGenerator.generateJsonData(SourceFile:170) + at com.atmire.statistics.mostpopular.JSONStatsMostPopularGenerator.generate(SourceFile:246) + at com.atmire.app.xmlui.aspect.statistics.JSONStatsMostPopular.generate(JSONStatsMostPopular.java:145) + at sun.reflect.GeneratedMethodAccessor296.invoke(Unknown Source) + at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.lang.reflect.Method.invoke(Method.java:498) + at org.apache.cocoon.core.container.spring.avalon.PoolableProxyHandler.invoke(PoolableProxyHandler.java:71) + at com.sun.proxy.$Proxy96.process(Unknown Source) + at org.apache.cocoon.components.treeprocessor.sitemap.ReadNode.invoke(ReadNode.java:94) + at org.apache.cocoon.components.treeprocessor.AbstractParentProcessingNode.invokeNodes(AbstractParentProcessingNode.java:55) + at org.apache.cocoon.components.treeprocessor.sitemap.MatchNode.invoke(MatchNode.java:87) + at org.apache.cocoon.components.treeprocessor.AbstractParentProcessingNode.invokeNodes(AbstractParentProcessingNode.java:55) + at org.apache.cocoon.components.treeprocessor.sitemap.MatchNode.invoke(MatchNode.java:87) + at org.apache.cocoon.components.treeprocessor.AbstractParentProcessingNode.invokeNodes(AbstractParentProcessingNode.java:78) + at org.apache.cocoon.components.treeprocessor.sitemap.PipelineNode.invoke(PipelineNode.java:143) + at org.apache.cocoon.components.treeprocessor.AbstractParentProcessingNode.invokeNodes(AbstractParentProcessingNode.java:78) + at org.apache.cocoon.components.treeprocessor.sitemap.PipelinesNode.invoke(PipelinesNode.java:81) + at org.apache.cocoon.components.treeprocessor.ConcreteTreeProcessor.process(ConcreteTreeProcessor.java:239) + at org.apache.cocoon.components.treeprocessor.ConcreteTreeProcessor.process(ConcreteTreeProcessor.java:171) + at org.apache.cocoon.components.treeprocessor.TreeProcessor.process(TreeProcessor.java:247) + at org.apache.cocoon.components.treeprocessor.sitemap.MountNode.invoke(MountNode.java:117) + at org.apache.cocoon.components.treeprocessor.AbstractParentProcessingNode.invokeNodes(AbstractParentProcessingNode.java:55) + at org.apache.cocoon.components.treeprocessor.sitemap.MatchNode.invoke(MatchNode.java:87) + at org.apache.cocoon.components.treeprocessor.AbstractParentProcessingNode.invokeNodes(AbstractParentProcessingNode.java:78) + at org.apache.cocoon.components.treeprocessor.sitemap.PipelineNode.invoke(PipelineNode.java:143) + at org.apache.cocoon.components.treeprocessor.AbstractParentProcessingNode.invokeNodes(AbstractParentProcessingNode.java:78) + at org.apache.cocoon.components.treeprocessor.sitemap.PipelinesNode.invoke(PipelinesNode.java:81) + at org.apache.cocoon.components.treeprocessor.ConcreteTreeProcessor.process(ConcreteTreeProcessor.java:239) + at org.apache.cocoon.components.treeprocessor.ConcreteTreeProcessor.process(ConcreteTreeProcessor.java:171) + at org.apache.cocoon.components.treeprocessor.TreeProcessor.process(TreeProcessor.java:247) + at org.apache.cocoon.servlet.RequestProcessor.process(RequestProcessor.java:351) + at org.apache.cocoon.servlet.RequestProcessor.service(RequestProcessor.java:169) + at org.apache.cocoon.sitemap.SitemapServlet.service(SitemapServlet.java:84) + at javax.servlet.http.HttpServlet.service(HttpServlet.java:727) + at org.apache.cocoon.servletservice.ServletServiceContext$PathDispatcher.forward(ServletServiceContext.java:468) + at org.apache.cocoon.servletservice.ServletServiceContext$PathDispatcher.forward(ServletServiceContext.java:443) + at org.apache.cocoon.servletservice.spring.ServletFactoryBean$ServiceInterceptor.invoke(ServletFactoryBean.java:264) + at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172) + at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:202) + at com.sun.proxy.$Proxy89.service(Unknown Source) + at org.dspace.springmvc.CocoonView.render(CocoonView.java:113) + at org.springframework.web.servlet.DispatcherServlet.render(DispatcherServlet.java:1180) + at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:950) + ... 35 more +``` + +- The first error I see in dspace.log this morning is: + +``` +2016-12-02 03:00:46,656 ERROR org.dspace.authority.AuthorityValueFinder @ anonymous::Error while retrieving AuthorityValue from solr:query\colon; id\colon;"b0b541c1-ec15-48bf-9209-6dbe8e338cdc" +org.apache.solr.client.solrj.SolrServerException: Server refused connection at: http://localhost:8081/solr/authority +``` + +- Looking through DSpace's solr log I see that about 20 seconds before this, there were a few 30+ KiB solr queries +- The last logs here right before Solr became unresponsive (and right after I restarted it five hours later) were: + +``` +2016-12-02 03:00:42,606 INFO org.apache.solr.core.SolrCore @ [statistics] webapp=/solr path=/select params={q=containerItem:72828+AND+type:0&shards=localhost:8081/solr/statistics-2010,localhost:8081/solr/statistics&fq=-isInternal:true&fq=-(author_mtdt:"CGIAR\+Institutional\+Learning\+and\+Change\+Initiative"++AND+subject_mtdt:"PARTNERSHIPS"+AND+subject_mtdt:"RESEARCH"+AND+subject_mtdt:"AGRICULTURE"+AND+subject_mtdt:"DEVELOPMENT"++AND+iso_mtdt:"en"+)&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=19 +2016-12-02 08:28:23,908 INFO org.apache.solr.servlet.SolrDispatchFilter @ SolrDispatchFilter.init() +``` + +- DSpace's own Solr logs don't give IP addresses, so I will have to enable Nginx's logging of `/solr` so I can see where this request came from +- I enabled logging of `/rest/` and I think I'll leave it on for good diff --git a/public/2015-11/index.html b/public/2015-11/index.html index bfcc0ee66..dae7e9ed3 100644 --- a/public/2015-11/index.html +++ b/public/2015-11/index.html @@ -313,6 +313,8 @@ db.statementpool = true

Recent Posts

    +
  1. December, 2016
    2. +
  2. November, 2016
  3. October, 2016
    4. @@ -321,8 +323,6 @@ db.statementpool = true
  4. August, 2016
    5. -
  5. July, 2016
    6. -
diff --git a/public/2015-12/index.html b/public/2015-12/index.html index 1c4da0eba..5ca3d2c95 100644 --- a/public/2015-12/index.html +++ b/public/2015-12/index.html @@ -331,6 +331,8 @@ $ curl -o /dev/null -s -w %{time_total}\\n https://cgspace.cgiar.org/rest/handle

Recent Posts

    +
  1. December, 2016
    2. +
  2. November, 2016
  3. October, 2016
    4. @@ -339,8 +341,6 @@ $ curl -o /dev/null -s -w %{time_total}\\n https://cgspace.cgiar.org/rest/handle
  4. August, 2016
    5. -
  5. July, 2016
    6. -
diff --git a/public/2016-01/index.html b/public/2016-01/index.html index 85d8e3a8f..7f3b110cd 100644 --- a/public/2016-01/index.html +++ b/public/2016-01/index.html @@ -241,6 +241,8 @@ $ find SimpleArchiveForBio/ -iname “*.pdf” -exec basename {} \; | so

Recent Posts

    +
  1. December, 2016
    2. +
  2. November, 2016
  3. October, 2016
    4. @@ -249,8 +251,6 @@ $ find SimpleArchiveForBio/ -iname “*.pdf” -exec basename {} \; | so
  4. August, 2016
    5. -
  5. July, 2016
    6. -
diff --git a/public/2016-02/index.html b/public/2016-02/index.html index 80dfe0256..c35e532a9 100644 --- a/public/2016-02/index.html +++ b/public/2016-02/index.html @@ -501,6 +501,8 @@ Bitstream: tést señora alimentación.pdf

Recent Posts

    +
  1. December, 2016
    2. +
  2. November, 2016
  3. October, 2016
    4. @@ -509,8 +511,6 @@ Bitstream: tést señora alimentación.pdf
  4. August, 2016
    5. -
  5. July, 2016
    6. -
diff --git a/public/2016-03/index.html b/public/2016-03/index.html index f648b69b3..1be8e6bad 100644 --- a/public/2016-03/index.html +++ b/public/2016-03/index.html @@ -394,6 +394,8 @@ Reinstall my local (Mac OS X) DSpace stack with Tomcat 7, PostgreSQL 9.3, and Ja

Recent Posts

    +
  1. December, 2016
    2. +
  2. November, 2016
  3. October, 2016
    4. @@ -402,8 +404,6 @@ Reinstall my local (Mac OS X) DSpace stack with Tomcat 7, PostgreSQL 9.3, and Ja
  4. August, 2016
    5. -
  5. July, 2016
    6. -
diff --git a/public/2016-04/index.html b/public/2016-04/index.html index beacb6ba2..84be8b89b 100644 --- a/public/2016-04/index.html +++ b/public/2016-04/index.html @@ -613,6 +613,8 @@ dspace.log.2016-04-27:7271

Recent Posts

    +
  1. December, 2016
    2. +
  2. November, 2016
  3. October, 2016
    4. @@ -621,8 +623,6 @@ dspace.log.2016-04-27:7271
  4. August, 2016
    5. -
  5. July, 2016
    6. -
diff --git a/public/2016-05/index.html b/public/2016-05/index.html index 74ed20d60..4761a9600 100644 --- a/public/2016-05/index.html +++ b/public/2016-05/index.html @@ -465,6 +465,8 @@ sys 0m20.540s

Recent Posts

    +
  1. December, 2016
    2. +
  2. November, 2016
  3. October, 2016
    4. @@ -473,8 +475,6 @@ sys 0m20.540s
  4. August, 2016
    5. -
  5. July, 2016
    6. -
diff --git a/public/2016-06/index.html b/public/2016-06/index.html index 0ebc1e184..831ee9346 100644 --- a/public/2016-06/index.html +++ b/public/2016-06/index.html @@ -509,6 +509,8 @@ $ ./delete-metadata-values.py -f dc.contributor.corporate -i Corporate-Authors-D

Recent Posts

    +
  1. December, 2016
    2. +
  2. November, 2016
  3. October, 2016
    4. @@ -517,8 +519,6 @@ $ ./delete-metadata-values.py -f dc.contributor.corporate -i Corporate-Authors-D
  4. August, 2016
    5. -
  5. July, 2016
    6. -
diff --git a/public/2016-07/index.html b/public/2016-07/index.html index 9199bed08..61079dd6a 100644 --- a/public/2016-07/index.html +++ b/public/2016-07/index.html @@ -435,6 +435,8 @@ discovery.index.authority.ignore-variants=true

Recent Posts

    +
  1. December, 2016
    2. +
  2. November, 2016
  3. October, 2016
    4. @@ -443,8 +445,6 @@ discovery.index.authority.ignore-variants=true
  4. August, 2016
    5. -
  5. July, 2016
    6. -
diff --git a/public/2016-08/index.html b/public/2016-08/index.html index d722736fb..90ec6e09d 100644 --- a/public/2016-08/index.html +++ b/public/2016-08/index.html @@ -505,6 +505,8 @@ $ JAVA_OPTS="-Dfile.encoding=UTF-8 -Xmx512m" /home/cgspace.cgiar.org/b

Recent Posts

    +
  1. December, 2016
    2. +
  2. November, 2016
  3. October, 2016
    4. @@ -513,8 +515,6 @@ $ JAVA_OPTS="-Dfile.encoding=UTF-8 -Xmx512m" /home/cgspace.cgiar.org/b
  4. August, 2016
    5. -
  5. July, 2016
    6. -
diff --git a/public/2016-09/index.html b/public/2016-09/index.html index 298eb7df5..da15ab312 100644 --- a/public/2016-09/index.html +++ b/public/2016-09/index.html @@ -801,6 +801,8 @@ $ ./delete-metadata-values.py -i ilrisubjects-delete-13.csv -f cg.subject.ilri -

Recent Posts

    +
  1. December, 2016
    2. +
  2. November, 2016
  3. October, 2016
    4. @@ -809,8 +811,6 @@ $ ./delete-metadata-values.py -i ilrisubjects-delete-13.csv -f cg.subject.ilri -
  4. August, 2016
    5. -
  5. July, 2016
    6. -
diff --git a/public/2016-10/index.html b/public/2016-10/index.html index fdde83a37..28c69bc86 100644 --- a/public/2016-10/index.html +++ b/public/2016-10/index.html @@ -444,6 +444,8 @@ dspace=# update metadatavalue set text_value = regexp_replace(text_value, 'http:

Recent Posts

    +
  1. December, 2016
    2. +
  2. November, 2016
  3. October, 2016
    4. @@ -452,8 +454,6 @@ dspace=# update metadatavalue set text_value = regexp_replace(text_value, 'http:
  4. August, 2016
    5. -
  5. July, 2016
    6. -
diff --git a/public/2016-11/index.html b/public/2016-11/index.html index 572e33070..465e681ec 100644 --- a/public/2016-11/index.html +++ b/public/2016-11/index.html @@ -30,7 +30,7 @@ - + @@ -653,6 +653,7 @@ org.dspace.discovery.SearchServiceException: Error executing query @@ -675,6 +676,8 @@ org.dspace.discovery.SearchServiceException: Error executing query

Recent Posts

    +
  1. December, 2016
    2. +
  2. November, 2016
  3. October, 2016
    4. @@ -683,8 +686,6 @@ org.dspace.discovery.SearchServiceException: Error executing query
  4. August, 2016
    5. -
  5. July, 2016
    6. -
diff --git a/public/2016-12/index.html b/public/2016-12/index.html new file mode 100644 index 000000000..a870a370a --- /dev/null +++ b/public/2016-12/index.html @@ -0,0 +1,312 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + December, 2016 | CGSpace Notes + + + + + + + + + + + + + + + + +
+
+ +
+
+ +
+
+

CGSpace Notes

+ +
+
+ +
+
+
+ + + + +
+
+

December, 2016

+

by Alan Orth in + +  + +

+
+ + +

2016-12-02

+ +
    +
  • CGSpace was down for five hours in the morning while I was sleeping
    • +
  • While looking in the logs for errors, I see tons of warnings about Atmire MQM:
    • +
+ +
2016-12-02 03:00:32,352 WARN  com.atmire.metadataquality.batchedit.BatchEditConsumer @ BatchEditConsumer should not have been given this kind of Subject in an event, skipping: org.dspace.event.Event(eventType=CREATE, SubjectType=BUNDLE, SubjectID=70316, ObjectType=(Unknown), ObjectID=-1, TimeStamp=1480647632305, dispatcher=1544803905, detail=[null], transactionID="TX157907838689377964651674089851855413607")
+2016-12-02 03:00:32,353 WARN  com.atmire.metadataquality.batchedit.BatchEditConsumer @ BatchEditConsumer should not have been given this kind of Subject in an event, skipping: org.dspace.event.Event(eventType=MODIFY_METADATA, SubjectType=BUNDLE, SubjectID =70316, ObjectType=(Unknown), ObjectID=-1, TimeStamp=1480647632309, dispatcher=1544803905, detail="dc.title", transactionID="TX157907838689377964651674089851855413607")
+2016-12-02 03:00:32,353 WARN  com.atmire.metadataquality.batchedit.BatchEditConsumer @ BatchEditConsumer should not have been given this kind of Subject in an event, skipping: org.dspace.event.Event(eventType=ADD, SubjectType=ITEM, SubjectID=80044, Object Type=BUNDLE, ObjectID=70316, TimeStamp=1480647632311, dispatcher=1544803905, detail="THUMBNAIL", transactionID="TX157907838689377964651674089851855413607")
+2016-12-02 03:00:32,353 WARN  com.atmire.metadataquality.batchedit.BatchEditConsumer @ BatchEditConsumer should not have been given this kind of Subject in an event, skipping: org.dspace.event.Event(eventType=ADD, SubjectType=BUNDLE, SubjectID=70316, Obje ctType=BITSTREAM, ObjectID=86715, TimeStamp=1480647632318, dispatcher=1544803905, detail="-1", transactionID="TX157907838689377964651674089851855413607")
+2016-12-02 03:00:32,353 WARN  com.atmire.metadataquality.batchedit.BatchEditConsumer @ BatchEditConsumer should not have been given this kind of Subject in an event, skipping: org.dspace.event.Event(eventType=MODIFY, SubjectType=ITEM, SubjectID=80044, ObjectType=(Unknown), ObjectID=-1, TimeStamp=1480647632351, dispatcher=1544803905, detail=[null], transactionID="TX157907838689377964651674089851855413607")
+
+ +
    +
  • I see thousands of them in the logs for the last few months, so it’s not related to the DSpace 5.5 upgrade
    • +
  • I’ve raised a ticket with Atmire to ask
    • +
  • Another worrying error from dspace.log is:
    • +
+ +
org.springframework.web.util.NestedServletException: Handler processing failed; nested exception is java.lang.NoSuchMethodError: com.atmire.statistics.generator.DSpaceObjectDatasetGenerator.toDatasetQuery(Lorg/dspace/core/Context;)Lcom/atmire/statistics/content/DatasetQuery;
+        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:972)
+        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:852)
+        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:882)
+        at org.springframework.web.servlet.FrameworkServlet.doPost(FrameworkServlet.java:789)
+        at javax.servlet.http.HttpServlet.service(HttpServlet.java:646)
+        at javax.servlet.http.HttpServlet.service(HttpServlet.java:727)
+        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:303)
+        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
+        at org.dspace.app.xmlui.cocoon.SetCharacterEncodingFilter.doFilter(SetCharacterEncodingFilter.java:111)
+        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
+        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
+        at org.dspace.app.xmlui.cocoon.DSpaceCocoonServletFilter.doFilter(DSpaceCocoonServletFilter.java:274)
+        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
+        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
+        at org.dspace.app.xmlui.cocoon.servlet.multipart.DSpaceMultipartFilter.doFilter(DSpaceMultipartFilter.java:119)
+        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
+        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
+        at org.dspace.utils.servlet.DSpaceWebappServletFilter.doFilter(DSpaceWebappServletFilter.java:78)
+        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
+        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
+        at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:220)
+        at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:122)
+        at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:501)
+        at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:170)
+        at com.googlecode.psiprobe.Tomcat70AgentValve.invoke(Tomcat70AgentValve.java:44)
+        at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:98)
+        at org.apache.catalina.valves.CrawlerSessionManagerValve.invoke(CrawlerSessionManagerValve.java:180)
+        at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:950)
+        at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116)
+        at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:408)
+        at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1041)
+        at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:607)
+        at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:313)
+        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
+        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
+        at java.lang.Thread.run(Thread.java:745)
+Caused by: java.lang.NoSuchMethodError: com.atmire.statistics.generator.DSpaceObjectDatasetGenerator.toDatasetQuery(Lorg/dspace/core/Context;)Lcom/atmire/statistics/content/DatasetQuery;
+        at com.atmire.statistics.generator.TopNDSODatasetGenerator.toDatasetQuery(SourceFile:39)
+        at com.atmire.statistics.display.StatisticsDataVisitsMultidata.createDataset(SourceFile:108)
+        at org.dspace.statistics.content.StatisticsDisplay.createDataset(SourceFile:384)
+        at org.dspace.statistics.content.StatisticsDisplay.getDataset(SourceFile:404)
+        at com.atmire.statistics.mostpopular.JSONStatsMostPopularGenerator.generateJsonData(SourceFile:170)
+        at com.atmire.statistics.mostpopular.JSONStatsMostPopularGenerator.generate(SourceFile:246)
+        at com.atmire.app.xmlui.aspect.statistics.JSONStatsMostPopular.generate(JSONStatsMostPopular.java:145)
+        at sun.reflect.GeneratedMethodAccessor296.invoke(Unknown Source)
+        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
+        at java.lang.reflect.Method.invoke(Method.java:498)
+        at org.apache.cocoon.core.container.spring.avalon.PoolableProxyHandler.invoke(PoolableProxyHandler.java:71)
+        at com.sun.proxy.$Proxy96.process(Unknown Source)
+        at org.apache.cocoon.components.treeprocessor.sitemap.ReadNode.invoke(ReadNode.java:94)
+        at org.apache.cocoon.components.treeprocessor.AbstractParentProcessingNode.invokeNodes(AbstractParentProcessingNode.java:55)
+        at org.apache.cocoon.components.treeprocessor.sitemap.MatchNode.invoke(MatchNode.java:87)
+        at org.apache.cocoon.components.treeprocessor.AbstractParentProcessingNode.invokeNodes(AbstractParentProcessingNode.java:55)
+        at org.apache.cocoon.components.treeprocessor.sitemap.MatchNode.invoke(MatchNode.java:87)
+        at org.apache.cocoon.components.treeprocessor.AbstractParentProcessingNode.invokeNodes(AbstractParentProcessingNode.java:78)
+        at org.apache.cocoon.components.treeprocessor.sitemap.PipelineNode.invoke(PipelineNode.java:143)
+        at org.apache.cocoon.components.treeprocessor.AbstractParentProcessingNode.invokeNodes(AbstractParentProcessingNode.java:78)
+        at org.apache.cocoon.components.treeprocessor.sitemap.PipelinesNode.invoke(PipelinesNode.java:81)
+        at org.apache.cocoon.components.treeprocessor.ConcreteTreeProcessor.process(ConcreteTreeProcessor.java:239)
+        at org.apache.cocoon.components.treeprocessor.ConcreteTreeProcessor.process(ConcreteTreeProcessor.java:171)
+        at org.apache.cocoon.components.treeprocessor.TreeProcessor.process(TreeProcessor.java:247)
+        at org.apache.cocoon.components.treeprocessor.sitemap.MountNode.invoke(MountNode.java:117)
+        at org.apache.cocoon.components.treeprocessor.AbstractParentProcessingNode.invokeNodes(AbstractParentProcessingNode.java:55)
+        at org.apache.cocoon.components.treeprocessor.sitemap.MatchNode.invoke(MatchNode.java:87)
+        at org.apache.cocoon.components.treeprocessor.AbstractParentProcessingNode.invokeNodes(AbstractParentProcessingNode.java:78)
+        at org.apache.cocoon.components.treeprocessor.sitemap.PipelineNode.invoke(PipelineNode.java:143)
+        at org.apache.cocoon.components.treeprocessor.AbstractParentProcessingNode.invokeNodes(AbstractParentProcessingNode.java:78)
+        at org.apache.cocoon.components.treeprocessor.sitemap.PipelinesNode.invoke(PipelinesNode.java:81)
+        at org.apache.cocoon.components.treeprocessor.ConcreteTreeProcessor.process(ConcreteTreeProcessor.java:239)
+        at org.apache.cocoon.components.treeprocessor.ConcreteTreeProcessor.process(ConcreteTreeProcessor.java:171)
+        at org.apache.cocoon.components.treeprocessor.TreeProcessor.process(TreeProcessor.java:247)
+        at org.apache.cocoon.servlet.RequestProcessor.process(RequestProcessor.java:351)
+        at org.apache.cocoon.servlet.RequestProcessor.service(RequestProcessor.java:169)
+        at org.apache.cocoon.sitemap.SitemapServlet.service(SitemapServlet.java:84)
+        at javax.servlet.http.HttpServlet.service(HttpServlet.java:727)
+        at org.apache.cocoon.servletservice.ServletServiceContext$PathDispatcher.forward(ServletServiceContext.java:468)
+        at org.apache.cocoon.servletservice.ServletServiceContext$PathDispatcher.forward(ServletServiceContext.java:443)
+        at org.apache.cocoon.servletservice.spring.ServletFactoryBean$ServiceInterceptor.invoke(ServletFactoryBean.java:264)
+        at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
+        at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:202)
+        at com.sun.proxy.$Proxy89.service(Unknown Source)
+        at org.dspace.springmvc.CocoonView.render(CocoonView.java:113)
+        at org.springframework.web.servlet.DispatcherServlet.render(DispatcherServlet.java:1180)
+        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:950)
+        ... 35 more
+
+ +
    +
  • The first error I see in dspace.log this morning is:
    • +
+ +
2016-12-02 03:00:46,656 ERROR org.dspace.authority.AuthorityValueFinder @ anonymous::Error while retrieving AuthorityValue from solr:query\colon; id\colon;"b0b541c1-ec15-48bf-9209-6dbe8e338cdc"
+org.apache.solr.client.solrj.SolrServerException: Server refused connection at: http://localhost:8081/solr/authority
+
+ +
    +
  • Looking through DSpace’s solr log I see that about 20 seconds before this, there were a few 30+ KiB solr queries
    • +
  • The last logs here right before Solr became unresponsive (and right after I restarted it five hours later) were:
    • +
+ +
2016-12-02 03:00:42,606 INFO  org.apache.solr.core.SolrCore @ [statistics] webapp=/solr path=/select params={q=containerItem:72828+AND+type:0&shards=localhost:8081/solr/statistics-2010,localhost:8081/solr/statistics&fq=-isInternal:true&fq=-(author_mtdt:"CGIAR\+Institutional\+Learning\+and\+Change\+Initiative"++AND+subject_mtdt:"PARTNERSHIPS"+AND+subject_mtdt:"RESEARCH"+AND+subject_mtdt:"AGRICULTURE"+AND+subject_mtdt:"DEVELOPMENT"++AND+iso_mtdt:"en"+)&rows=0&wt=javabin&version=2} hits=0 status=0 QTime=19
+2016-12-02 08:28:23,908 INFO  org.apache.solr.servlet.SolrDispatchFilter @ SolrDispatchFilter.init()
+
+ +
    +
  • DSpace’s own Solr logs don’t give IP addresses, so I will have to enable Nginx’s logging of /solr so I can see where this request came from
    • +
  • I enabled logging of /rest/ and I think I’ll leave it on for good
    • +
+ + + + +
+ + + + + +
+ + + + +
+
+ + + + + + diff --git a/public/index.html b/public/index.html index 66ed77e3d..0d70a08b8 100644 --- a/public/index.html +++ b/public/index.html @@ -13,7 +13,7 @@ - + @@ -91,6 +91,22 @@ +
+
+

December, 2016

+

by Alan Orth in + +  + +

+
+ 2016-12-02 CGSpace was down for five hours in the morning while I was sleeping While looking in the logs for errors, I see tons of warnings about Atmire MQM: 2016-12-02 03:00:32,352 WARN com.atmire.metadataquality.batchedit.BatchEditConsumer @ BatchEditConsumer should not have been given this kind of Subject in an event, skipping: org.dspace.event.Event(eventType=CREATE, SubjectType=BUNDLE, SubjectID=70316, ObjectType=(Unknown), ObjectID=-1, TimeStamp=1480647632305, dispatcher=1544803905, detail=[null], transactionID="TX157907838689377964651674089851855413607") 2016-12-02 03:00:32,353 WARN com.atmire.metadataquality.batchedit.BatchEditConsumer @ BatchEditConsumer should not have been given this kind of Subject in an event, skipping: org. + Read more → +
+ + + +

November, 2016

@@ -323,37 +339,6 @@ dspacetest=# select text_value from metadatavalue where metadata_field_id=3 and -
-
-

February, 2016

-

by Alan Orth in - -  - -

-
-

2016-02-05

- -
    -
  • Looking at some DAGRIS data for Abenet Yabowork
    • -
  • Lots of issues with spaces, newlines, etc causing the import to fail
    • -
  • I noticed we have a very interesting list of countries on CGSpace:
    • -
- -

CGSpace country list

- -
    -
  • Not only are there 49,000 countries, we have some blanks (25)…
    • -
  • Also, lots of things like “COTE D`LVOIRE” and “COTE D IVOIRE”
    • -
- -

- Read more → -
- - - -