diff --git a/content/posts/2019-03.md b/content/posts/2019-03.md index 7475a31a9..1d896bac8 100644 --- a/content/posts/2019-03.md +++ b/content/posts/2019-03.md @@ -182,4 +182,130 @@ $ grep -oE '201[89]' /tmp/*.csv | sort -u - Run all system updates and reboot linode20 - Follow up with Felix from Earlham to see if he's done testing DSpace Test with COPO so I can re-sync the server from CGSpace +## 2019-03-15 + +- CGSpace (linode18) has the blank page error again +- I'm not sure if it's related, but I see the following error in DSpace's log: + +``` +2019-03-15 14:09:32,685 ERROR org.dspace.storage.rdbms.DatabaseManager @ SQL QueryTable Error - +java.sql.SQLException: Connection org.postgresql.jdbc.PgConnection@55ba10b5 is closed. + at org.apache.tomcat.dbcp.dbcp.DelegatingConnection.checkOpen(DelegatingConnection.java:398) + at org.apache.tomcat.dbcp.dbcp.DelegatingConnection.prepareStatement(DelegatingConnection.java:279) + at org.apache.tomcat.dbcp.dbcp.PoolingDataSource$PoolGuardConnectionWrapper.prepareStatement(PoolingDataSource.java:313) + at org.dspace.storage.rdbms.DatabaseManager.queryTable(DatabaseManager.java:220) + at org.dspace.authorize.AuthorizeManager.getPolicies(AuthorizeManager.java:612) + at org.dspace.content.crosswalk.METSRightsCrosswalk.disseminateElement(METSRightsCrosswalk.java:154) + at org.dspace.content.crosswalk.METSRightsCrosswalk.disseminateElement(METSRightsCrosswalk.java:300) +``` + +- Interestingly, I see a pattern of these errors increasing, with single and double digit numbers over the past month, but spikes of over 1,000 today, yesterday, and on 2019-03-08, which was exactly the first time we saw this blank page error recently + +``` +# grep -c 'SQL QueryTable Error' dspace.log.2019-0* +dspace.log.2019-02-01:6 +dspace.log.2019-02-02:1 +dspace.log.2019-02-03:0 +dspace.log.2019-02-04:4 +dspace.log.2019-02-05:4 +dspace.log.2019-02-06:3 +dspace.log.2019-02-07:0 +dspace.log.2019-02-08:2 +dspace.log.2019-02-09:0 +dspace.log.2019-02-10:0 +dspace.log.2019-02-11:1 +dspace.log.2019-02-12:2 +dspace.log.2019-02-13:1 +dspace.log.2019-02-14:74 +dspace.log.2019-02-15:47 +dspace.log.2019-02-16:44 +dspace.log.2019-02-17:37 +dspace.log.2019-02-18:191 +dspace.log.2019-02-19:51 +dspace.log.2019-02-20:32 +dspace.log.2019-02-21:64 +dspace.log.2019-02-22:48 +dspace.log.2019-02-23:32 +dspace.log.2019-02-24:25 +dspace.log.2019-02-25:53 +dspace.log.2019-02-26:63 +dspace.log.2019-02-27:28 +dspace.log.2019-02-28:31 +dspace.log.2019-03-01:29 +dspace.log.2019-03-02:24 +dspace.log.2019-03-03:41 +dspace.log.2019-03-04:25 +dspace.log.2019-03-05:48 +dspace.log.2019-03-06:48 +dspace.log.2019-03-07:42 +dspace.log.2019-03-08:1781 +dspace.log.2019-03-09:22 +dspace.log.2019-03-10:23 +dspace.log.2019-03-11:52 +dspace.log.2019-03-12:474 +dspace.log.2019-03-13:37 +dspace.log.2019-03-14:924 +dspace.log.2019-03-15:770 +``` + +- There are not too many connections currently in PostgreSQL: + +``` +$ psql -c 'select * from pg_stat_activity' | grep -o -E '(dspaceWeb|dspaceApi|dspaceCli)' | sort | uniq -c + 6 dspaceApi + 10 dspaceCli + 15 dspaceWeb +``` + +- I didn't see anything interesting in the PostgreSQL logs, though this stack trace from the Tomcat logs (in the systemd journal) from earlier today *might* be related? + +``` +SEVERE: Servlet.service() for servlet [spring] in context with path [] threw exception [org.springframework.web.util.NestedServletException: Request processing failed; nested exception is java.util.EmptyStackException] with root cause +java.util.EmptyStackException + at java.util.Stack.peek(Stack.java:102) + at java.util.Stack.pop(Stack.java:84) + at org.apache.cocoon.callstack.CallStack.leave(CallStack.java:54) + at org.apache.cocoon.servletservice.CallStackHelper.leaveServlet(CallStackHelper.java:85) + at org.apache.cocoon.servletservice.ServletServiceContext$PathDispatcher.forward(ServletServiceContext.java:484) + at org.apache.cocoon.servletservice.ServletServiceContext$PathDispatcher.forward(ServletServiceContext.java:443) + at org.apache.cocoon.servletservice.spring.ServletFactoryBean$ServiceInterceptor.invoke(ServletFactoryBean.java:264) + at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172) + at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:202) + at com.sun.proxy.$Proxy90.service(Unknown Source) + at org.dspace.springmvc.CocoonView.render(CocoonView.java:113) + at org.springframework.web.servlet.DispatcherServlet.render(DispatcherServlet.java:1180) + at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:950) + at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:852) + at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:882) + at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:778) + at javax.servlet.http.HttpServlet.service(HttpServlet.java:624) + at javax.servlet.http.HttpServlet.service(HttpServlet.java:731) + at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:303) + at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) + at org.dspace.rdf.negotiation.NegotiationFilter.doFilter(NegotiationFilter.java:59) + at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) + at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) + at org.dspace.utils.servlet.DSpaceWebappServletFilter.doFilter(DSpaceWebappServletFilter.java:78) + at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241) + at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208) + at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:219) + at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:110) + at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:494) + at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:169) + at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:104) + at org.apache.catalina.valves.CrawlerSessionManagerValve.invoke(CrawlerSessionManagerValve.java:234) + at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:1025) + at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116) + at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:445) + at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1137) + at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:637) + at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:317) + at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) + at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) + at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61) + at java.lang.Thread.run(Thread.java:748) +``` + +- For now I will just restart Tomcat... + diff --git a/docs/2019-03/index.html b/docs/2019-03/index.html index b92ba8ea5..55bdf6c67 100644 --- a/docs/2019-03/index.html +++ b/docs/2019-03/index.html @@ -25,7 +25,7 @@ I think I will need to ask Udana to re-copy and paste the abstracts with more ca - + @@ -55,9 +55,9 @@ I think I will need to ask Udana to re-copy and paste the abstracts with more ca "@type": "BlogPosting", "headline": "March, 2019", "url": "https://alanorth.github.io/cgspace-notes/2019-03/", - "wordCount": "1471", + "wordCount": "1790", "datePublished": "2019-03-01T12:16:30+01:00", - "dateModified": "2019-03-14T20:18:29+02:00", + "dateModified": "2019-03-14T22:15:10+02:00", "author": { "@type": "Person", "name": "Alan Orth" @@ -352,6 +352,138 @@ done
  • Follow up with Felix from Earlham to see if he’s done testing DSpace Test with COPO so I can re-sync the server from CGSpace
  • +

    2019-03-15

    + + + +
    2019-03-15 14:09:32,685 ERROR org.dspace.storage.rdbms.DatabaseManager @ SQL QueryTable Error -
    +java.sql.SQLException: Connection org.postgresql.jdbc.PgConnection@55ba10b5 is closed.
    +        at org.apache.tomcat.dbcp.dbcp.DelegatingConnection.checkOpen(DelegatingConnection.java:398)
    +        at org.apache.tomcat.dbcp.dbcp.DelegatingConnection.prepareStatement(DelegatingConnection.java:279)
    +        at org.apache.tomcat.dbcp.dbcp.PoolingDataSource$PoolGuardConnectionWrapper.prepareStatement(PoolingDataSource.java:313)
    +        at org.dspace.storage.rdbms.DatabaseManager.queryTable(DatabaseManager.java:220)
    +        at org.dspace.authorize.AuthorizeManager.getPolicies(AuthorizeManager.java:612)
    +        at org.dspace.content.crosswalk.METSRightsCrosswalk.disseminateElement(METSRightsCrosswalk.java:154)
    +        at org.dspace.content.crosswalk.METSRightsCrosswalk.disseminateElement(METSRightsCrosswalk.java:300)
    +
    + + + +
    # grep -c 'SQL QueryTable Error' dspace.log.2019-0*
    +dspace.log.2019-02-01:6
    +dspace.log.2019-02-02:1
    +dspace.log.2019-02-03:0
    +dspace.log.2019-02-04:4
    +dspace.log.2019-02-05:4
    +dspace.log.2019-02-06:3
    +dspace.log.2019-02-07:0
    +dspace.log.2019-02-08:2
    +dspace.log.2019-02-09:0
    +dspace.log.2019-02-10:0
    +dspace.log.2019-02-11:1
    +dspace.log.2019-02-12:2
    +dspace.log.2019-02-13:1
    +dspace.log.2019-02-14:74
    +dspace.log.2019-02-15:47
    +dspace.log.2019-02-16:44
    +dspace.log.2019-02-17:37
    +dspace.log.2019-02-18:191
    +dspace.log.2019-02-19:51
    +dspace.log.2019-02-20:32
    +dspace.log.2019-02-21:64
    +dspace.log.2019-02-22:48
    +dspace.log.2019-02-23:32
    +dspace.log.2019-02-24:25
    +dspace.log.2019-02-25:53
    +dspace.log.2019-02-26:63
    +dspace.log.2019-02-27:28
    +dspace.log.2019-02-28:31
    +dspace.log.2019-03-01:29
    +dspace.log.2019-03-02:24
    +dspace.log.2019-03-03:41
    +dspace.log.2019-03-04:25
    +dspace.log.2019-03-05:48
    +dspace.log.2019-03-06:48
    +dspace.log.2019-03-07:42
    +dspace.log.2019-03-08:1781
    +dspace.log.2019-03-09:22
    +dspace.log.2019-03-10:23
    +dspace.log.2019-03-11:52
    +dspace.log.2019-03-12:474
    +dspace.log.2019-03-13:37
    +dspace.log.2019-03-14:924
    +dspace.log.2019-03-15:770
    +
    + + + +
    $ psql -c 'select * from pg_stat_activity' | grep -o -E '(dspaceWeb|dspaceApi|dspaceCli)' | sort | uniq -c
    +      6 dspaceApi
    +     10 dspaceCli
    +     15 dspaceWeb
    +
    + + + +
    SEVERE: Servlet.service() for servlet [spring] in context with path [] threw exception [org.springframework.web.util.NestedServletException: Request processing failed; nested exception is java.util.EmptyStackException] with root cause
    +java.util.EmptyStackException
    +        at java.util.Stack.peek(Stack.java:102)
    +        at java.util.Stack.pop(Stack.java:84)
    +        at org.apache.cocoon.callstack.CallStack.leave(CallStack.java:54)
    +        at org.apache.cocoon.servletservice.CallStackHelper.leaveServlet(CallStackHelper.java:85)
    +        at org.apache.cocoon.servletservice.ServletServiceContext$PathDispatcher.forward(ServletServiceContext.java:484)
    +        at org.apache.cocoon.servletservice.ServletServiceContext$PathDispatcher.forward(ServletServiceContext.java:443)
    +        at org.apache.cocoon.servletservice.spring.ServletFactoryBean$ServiceInterceptor.invoke(ServletFactoryBean.java:264)
    +        at org.springframework.aop.framework.ReflectiveMethodInvocation.proceed(ReflectiveMethodInvocation.java:172)
    +        at org.springframework.aop.framework.JdkDynamicAopProxy.invoke(JdkDynamicAopProxy.java:202)
    +        at com.sun.proxy.$Proxy90.service(Unknown Source)
    +        at org.dspace.springmvc.CocoonView.render(CocoonView.java:113)
    +        at org.springframework.web.servlet.DispatcherServlet.render(DispatcherServlet.java:1180)
    +        at org.springframework.web.servlet.DispatcherServlet.doDispatch(DispatcherServlet.java:950)
    +        at org.springframework.web.servlet.DispatcherServlet.doService(DispatcherServlet.java:852)
    +        at org.springframework.web.servlet.FrameworkServlet.processRequest(FrameworkServlet.java:882)
    +        at org.springframework.web.servlet.FrameworkServlet.doGet(FrameworkServlet.java:778)
    +        at javax.servlet.http.HttpServlet.service(HttpServlet.java:624)
    +        at javax.servlet.http.HttpServlet.service(HttpServlet.java:731)
    +        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:303)
    +        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
    +        at org.dspace.rdf.negotiation.NegotiationFilter.doFilter(NegotiationFilter.java:59)
    +        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
    +        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
    +        at org.dspace.utils.servlet.DSpaceWebappServletFilter.doFilter(DSpaceWebappServletFilter.java:78)
    +        at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:241)
    +        at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:208)
    +        at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:219)
    +        at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:110)
    +        at org.apache.catalina.authenticator.AuthenticatorBase.invoke(AuthenticatorBase.java:494)
    +        at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:169)
    +        at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:104)
    +        at org.apache.catalina.valves.CrawlerSessionManagerValve.invoke(CrawlerSessionManagerValve.java:234)
    +        at org.apache.catalina.valves.AccessLogValve.invoke(AccessLogValve.java:1025)
    +        at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:116)
    +        at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:445)
    +        at org.apache.coyote.http11.AbstractHttp11Processor.process(AbstractHttp11Processor.java:1137)
    +        at org.apache.coyote.AbstractProtocol$AbstractConnectionHandler.process(AbstractProtocol.java:637)
    +        at org.apache.tomcat.util.net.JIoEndpoint$SocketProcessor.run(JIoEndpoint.java:317)
    +        at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
    +        at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
    +        at org.apache.tomcat.util.threads.TaskThread$WrappingRunnable.run(TaskThread.java:61)
    +        at java.lang.Thread.run(Thread.java:748)
    +
    + + + diff --git a/docs/sitemap.xml b/docs/sitemap.xml index 60f5431bc..1abeb9579 100644 --- a/docs/sitemap.xml +++ b/docs/sitemap.xml @@ -4,7 +4,7 @@ https://alanorth.github.io/cgspace-notes/2019-03/ - 2019-03-14T20:18:29+02:00 + 2019-03-14T22:15:10+02:00 @@ -214,7 +214,7 @@ https://alanorth.github.io/cgspace-notes/ - 2019-03-14T20:18:29+02:00 + 2019-03-14T22:15:10+02:00 0 @@ -225,7 +225,7 @@ https://alanorth.github.io/cgspace-notes/tags/notes/ - 2019-03-14T20:18:29+02:00 + 2019-03-14T22:15:10+02:00 0 @@ -237,13 +237,13 @@ https://alanorth.github.io/cgspace-notes/posts/ - 2019-03-14T20:18:29+02:00 + 2019-03-14T22:15:10+02:00 0 https://alanorth.github.io/cgspace-notes/tags/ - 2019-03-14T20:18:29+02:00 + 2019-03-14T22:15:10+02:00 0