From a50ad3ac768c63692b7f65260db53de0a57891d4 Mon Sep 17 00:00:00 2001 From: Alan Orth Date: Mon, 16 Nov 2020 10:54:00 +0200 Subject: [PATCH] Regenerate docs --- docs/2015-11/index.html | 10 +-- docs/2015-12/index.html | 10 +-- docs/2016-01/index.html | 10 +-- docs/2016-02/index.html | 10 +-- docs/2016-03/index.html | 10 +-- docs/2016-04/index.html | 10 +-- docs/2016-05/index.html | 10 +-- docs/2016-06/index.html | 10 +-- docs/2016-07/index.html | 10 +-- docs/2016-08/index.html | 10 +-- docs/2016-09/index.html | 10 +-- docs/2016-10/index.html | 10 +-- docs/2016-11/index.html | 10 +-- docs/2016-12/index.html | 10 +-- docs/2017-01/index.html | 10 +-- docs/2017-02/index.html | 10 +-- docs/2017-03/index.html | 10 +-- docs/2017-04/index.html | 10 +-- docs/2017-05/index.html | 10 +-- docs/2017-06/index.html | 10 +-- docs/2017-07/index.html | 10 +-- docs/2017-08/index.html | 10 +-- docs/2017-09/index.html | 10 +-- docs/2017-10/index.html | 10 +-- docs/2017-11/index.html | 10 +-- docs/2017-12/index.html | 10 +-- docs/2018-01/index.html | 10 +-- docs/2018-02/index.html | 10 +-- docs/2018-03/index.html | 10 +-- docs/2018-04/index.html | 10 +-- docs/2018-05/index.html | 10 +-- docs/2018-06/index.html | 10 +-- docs/2018-07/index.html | 10 +-- docs/2018-08/index.html | 10 +-- docs/2018-09/index.html | 10 +-- docs/2018-10/index.html | 10 +-- docs/2018-11/index.html | 10 +-- docs/2018-12/index.html | 10 +-- docs/2019-01/index.html | 10 +-- docs/2019-02/index.html | 10 +-- docs/2019-03/index.html | 10 +-- docs/2019-04/index.html | 10 +-- docs/2019-05/index.html | 10 +-- docs/2019-06/index.html | 10 +-- docs/2019-07/index.html | 10 +-- docs/2019-08/index.html | 10 +-- docs/2019-09/index.html | 10 +-- docs/2019-10/index.html | 10 +-- docs/2019-11/index.html | 10 +-- docs/2019-12/index.html | 10 +-- docs/2020-01/index.html | 10 +-- docs/2020-02/index.html | 10 +-- docs/2020-03/index.html | 10 +-- docs/2020-04/index.html | 10 +-- docs/2020-05/index.html | 10 +-- docs/2020-06/index.html | 10 +-- docs/2020-07/index.html | 10 +-- docs/2020-08/index.html | 10 +-- docs/2020-09/index.html | 10 +-- docs/2020-10/index.html | 18 +++--- docs/2020-11/index.html | 62 ++++++++++++++++--- docs/404.html | 6 +- docs/categories/index.html | 8 +-- docs/categories/notes/index.html | 8 +-- docs/categories/notes/page/2/index.html | 8 +-- docs/categories/notes/page/3/index.html | 8 +-- docs/categories/notes/page/4/index.html | 8 +-- docs/cgiar-library-migration/index.html | 10 +-- docs/cgspace-cgcorev2-migration/index.html | 10 +-- ...6c5dbe3f19ac4aa6bfb066f1243045b8f6061e.css | 6 -- ...a38b91c8bb5fb929e77b447d5f77c7285221ad.css | 6 ++ docs/index.html | 10 ++- ...26ff44a556b0808cb98a9db4008752110ed964b.js | 2 + ...bd496475309fc48e2b1d052c5c09e6facffcb0f.js | 1 - docs/page/2/index.html | 10 ++- docs/page/3/index.html | 10 ++- docs/page/4/index.html | 10 ++- docs/page/5/index.html | 10 ++- docs/page/6/index.html | 10 ++- docs/page/7/index.html | 10 ++- docs/posts/index.html | 10 ++- docs/posts/page/2/index.html | 10 ++- docs/posts/page/3/index.html | 10 ++- docs/posts/page/4/index.html | 10 ++- docs/posts/page/5/index.html | 10 ++- docs/posts/page/6/index.html | 10 ++- docs/posts/page/7/index.html | 10 ++- docs/sitemap.xml | 12 ++-- docs/tags/index.html | 6 +- docs/tags/migration/index.html | 6 +- docs/tags/notes/index.html | 6 +- docs/tags/notes/page/2/index.html | 6 +- docs/tags/notes/page/3/index.html | 6 +- 93 files changed, 464 insertions(+), 469 deletions(-) delete mode 100644 docs/css/style.6da5c906cc7a8fbb93f31cd2316c5dbe3f19ac4aa6bfb066f1243045b8f6061e.css create mode 100644 docs/css/style.d20c61b183eb27beb5b2c48f70a38b91c8bb5fb929e77b447d5f77c7285221ad.css create mode 100644 docs/js/fontawesome.min.4ed405d7c7002b970d34cbe6026ff44a556b0808cb98a9db4008752110ed964b.js delete mode 100644 docs/js/fontawesome.min.f3d2a1f5980bab30ddd0d8cadbd496475309fc48e2b1d052c5c09e6facffcb0f.js diff --git a/docs/2015-11/index.html b/docs/2015-11/index.html index ab5b98eaf..b6d0507b8 100644 --- a/docs/2015-11/index.html +++ b/docs/2015-11/index.html @@ -61,20 +61,18 @@ $ psql -c 'SELECT * from pg_stat_activity;' | grep idle | grep -c cgspac - + - + - - @@ -111,7 +109,9 @@ $ psql -c 'SELECT * from pg_stat_activity;' | grep idle | grep -c cgspac

November, 2015

-

December, 2015

-

January, 2016

-

February, 2016

-

March, 2016

-

April, 2016

-

May, 2016

-

June, 2016

-

July, 2016

-

August, 2016

-

September, 2016

-

October, 2016

-

November, 2016

-

December, 2016

-

January, 2017

-

February, 2017

-

March, 2017

-

April, 2017

-

May, 2017

-

June, 2017

-

July, 2017

-

August, 2017

-

September, 2017

-

October, 2017

-

November, 2017

-

December, 2017

-

January, 2018

-

February, 2018

-

March, 2018

-

April, 2018

-

May, 2018

-

June, 2018

-

July, 2018

-

August, 2018

-

September, 2018

-

October, 2018

-

November, 2018

-

December, 2018

-

January, 2019

-

February, 2019

-

March, 2019

-

April, 2019

-

May, 2019

-

June, 2019

-

July, 2019

-

August, 2019

-

September, 2019

-

October, 2019

-

November, 2019

-

December, 2019

-

January, 2020

-

February, 2020

-

March, 2020

-

April, 2020

-

May, 2020

-

June, 2020

-

July, 2020

-

August, 2020

-

September, 2020

-

October, 2020

-
  • So basically, as I saw at this same step in 2020-05, there are some documents that have IDs that have not been converted to UUID, and have not been labeled as “unmigrated” either…
      -
    • I see there are about 217,000 of them, 99% of which are of type: 5 which is “search”
    • +
    • I see there are about 217,000 of them, 99% of which are of type: 5 which is “site”
    • I purged them:
  • @@ -960,7 +960,7 @@ java.lang.OutOfMemoryError: Java heap space
    • I had the same problem when I processed the statistics-2018 core in 2020-07 and 2020-08
        -
      • I will try to purge some unmigrated records (around 460,000), most of which are of type: 5 (search) so not relevant to our views and downloads anyways:
      • +
      • I will try to purge some unmigrated records (around 460,000), most of which are of type: 5 (site) so not relevant to our views and downloads anyways:
    diff --git a/docs/2020-11/index.html b/docs/2020-11/index.html index 9e80fde15..94423a81e 100644 --- a/docs/2020-11/index.html +++ b/docs/2020-11/index.html @@ -17,7 +17,7 @@ So far we’ve spent at least fifty hours to process the statistics and stat - + @@ -39,9 +39,9 @@ So far we’ve spent at least fifty hours to process the statistics and stat "@type": "BlogPosting", "headline": "November, 2020", "url": "https://alanorth.github.io/cgspace-notes/2020-11/", - "wordCount": "1565", + "wordCount": "1762", "datePublished": "2020-11-01T13:11:54+02:00", - "dateModified": "2020-11-13T09:22:18+02:00", + "dateModified": "2020-11-16T10:53:45+02:00", "author": { "@type": "Person", "name": "Alan Orth" @@ -59,20 +59,18 @@ So far we’ve spent at least fifty hours to process the statistics and stat - + - + - - @@ -109,7 +107,9 @@ So far we’ve spent at least fifty hours to process the statistics and stat

    November, 2020

    -
  • Start a full Discovery re-indexing:
$ time chrt -b 0 ionice -c2 -n7 nice -n19 dspace index-discovery -b
+
+real    211m30.726s
+user    134m40.124s
+sys     2m17.979s
+
    +
  • Towards the end of the indexing there were a few dozen of these messages:
  • +
+
2020-11-15 13:23:21,685 INFO  com.atmire.dspace.discovery.service.AtmireSolrService @ Removed Item: null from Index
 
  • I updated all the Ansible infrastructure and DSpace branches to be the DSpace 6 ones
  • I will wait until the Discovery indexing is finished to start doing the Solr statistics migration
  • +
  • I tested the email functionality and it seems to need more configuration:
- +
$ dspace test-email
+
+About to send test email:
+ - To: blah@cgiar.org
+ - Subject: DSpace test email
+ - Server: smtp.office365.com
+
+Error sending email:
+ - Error: com.sun.mail.smtp.SMTPSendFailedException: 451 5.7.3 STARTTLS is required to send mail [AM4PR0701CA0003.eurprd07.prod.outlook.com]
+
    +
  • I copied the mail.extraproperties = mail.smtp.starttls.enable=true setting from the old DSpace 5 dspace.cfg and now the emails are working
  • +
  • After the Discovery indexing finished I started processing the Solr stats one core and 2.5 million records at a time:
  • +
+
$ export JAVA_OPTS='-Dfile.encoding=UTF-8 -Xmx2048m'
+$ chrt -b 0 dspace solr-upgrade-statistics-6x -n 2500000 -i statistics
+
    +
  • After about 6,000,000 records I got the same error that I’ve gotten every time I test this migration process:
  • +
+
Exception: Error while creating field 'p_group_id{type=uuid,properties=indexed,stored,multiValued}' from value '10'
+org.apache.solr.client.solrj.impl.HttpSolrServer$RemoteSolrException: Error while creating field 'p_group_id{type=uuid,properties=indexed,stored,multiValued}' from value '10'
+        at org.apache.solr.client.solrj.impl.HttpSolrServer.executeMethod(HttpSolrServer.java:552)
+        at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:210)
+        at org.apache.solr.client.solrj.impl.HttpSolrServer.request(HttpSolrServer.java:206)
+        at org.apache.solr.client.solrj.request.AbstractUpdateRequest.process(AbstractUpdateRequest.java:124)
+        at org.apache.solr.client.solrj.SolrServer.add(SolrServer.java:68)
+        at org.apache.solr.client.solrj.SolrServer.add(SolrServer.java:54)
+        at org.dspace.util.SolrUpgradePre6xStatistics.batchUpdateStats(SolrUpgradePre6xStatistics.java:161)
+        at org.dspace.util.SolrUpgradePre6xStatistics.run(SolrUpgradePre6xStatistics.java:456)
+        at org.dspace.util.SolrUpgradePre6xStatistics.main(SolrUpgradePre6xStatistics.java:365)
+        at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
+        at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
+        at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
+        at java.lang.reflect.Method.invoke(Method.java:498)
+        at org.dspace.app.launcher.ScriptLauncher.runOneCommand(ScriptLauncher.java:229)
+        at org.dspace.app.launcher.ScriptLauncher.main(ScriptLauncher.java:81)
+
diff --git a/docs/404.html b/docs/404.html index b5c67e788..c7e106e26 100644 --- a/docs/404.html +++ b/docs/404.html @@ -28,20 +28,18 @@ - + - + - - diff --git a/docs/categories/index.html b/docs/categories/index.html index d9beb3f92..4bb1cb0cf 100644 --- a/docs/categories/index.html +++ b/docs/categories/index.html @@ -9,7 +9,7 @@ - + @@ -28,12 +28,12 @@ - + - + @@ -41,8 +41,6 @@ - - diff --git a/docs/categories/notes/index.html b/docs/categories/notes/index.html index 7f49da826..b2ae43640 100644 --- a/docs/categories/notes/index.html +++ b/docs/categories/notes/index.html @@ -9,7 +9,7 @@ - + @@ -28,12 +28,12 @@ - + - + @@ -41,8 +41,6 @@ - - diff --git a/docs/categories/notes/page/2/index.html b/docs/categories/notes/page/2/index.html index 5fcbc420e..7d3043e97 100644 --- a/docs/categories/notes/page/2/index.html +++ b/docs/categories/notes/page/2/index.html @@ -9,7 +9,7 @@ - + @@ -28,12 +28,12 @@ - + - + @@ -41,8 +41,6 @@ - - diff --git a/docs/categories/notes/page/3/index.html b/docs/categories/notes/page/3/index.html index 8cda71291..cd22e7fa4 100644 --- a/docs/categories/notes/page/3/index.html +++ b/docs/categories/notes/page/3/index.html @@ -9,7 +9,7 @@ - + @@ -28,12 +28,12 @@ - + - + @@ -41,8 +41,6 @@ - - diff --git a/docs/categories/notes/page/4/index.html b/docs/categories/notes/page/4/index.html index 00e7a4eb9..b7559947b 100644 --- a/docs/categories/notes/page/4/index.html +++ b/docs/categories/notes/page/4/index.html @@ -9,7 +9,7 @@ - + @@ -28,12 +28,12 @@ - + - + @@ -41,8 +41,6 @@ - - diff --git a/docs/cgiar-library-migration/index.html b/docs/cgiar-library-migration/index.html index 850efecc4..0de16caf0 100644 --- a/docs/cgiar-library-migration/index.html +++ b/docs/cgiar-library-migration/index.html @@ -46,20 +46,18 @@ - + - + - - @@ -96,7 +94,9 @@

CGIAR Library Migration

-

CGSpace CG Core v2 Migration

-