From 24e5907937516565b1c0dc8b88da25dc5acf7bb2 Mon Sep 17 00:00:00 2001 From: Alan Orth Date: Mon, 26 Apr 2021 15:58:48 +0300 Subject: [PATCH] Update notes for 2021-04-26 --- content/posts/2021-04.md | 150 ++++++++++++++++++++++++ docs/2021-04/index.html | 146 ++++++++++++++++++++++- docs/categories/index.html | 2 +- docs/categories/notes/index.html | 2 +- docs/categories/notes/page/2/index.html | 2 +- docs/categories/notes/page/3/index.html | 2 +- docs/categories/notes/page/4/index.html | 2 +- docs/categories/notes/page/5/index.html | 2 +- docs/index.html | 2 +- docs/page/2/index.html | 2 +- docs/page/3/index.html | 2 +- docs/page/4/index.html | 2 +- docs/page/5/index.html | 2 +- docs/page/6/index.html | 2 +- docs/page/7/index.html | 2 +- docs/posts/index.html | 2 +- docs/posts/page/2/index.html | 2 +- docs/posts/page/3/index.html | 2 +- docs/posts/page/4/index.html | 2 +- docs/posts/page/5/index.html | 2 +- docs/posts/page/6/index.html | 2 +- docs/posts/page/7/index.html | 2 +- docs/sitemap.xml | 10 +- 23 files changed, 317 insertions(+), 29 deletions(-) diff --git a/content/posts/2021-04.md b/content/posts/2021-04.md index abd6f810b..b51f5f26d 100644 --- a/content/posts/2021-04.md +++ b/content/posts/2021-04.md @@ -747,4 +747,154 @@ $ chrt -b 0 dspace dsrun com.atmire.statistics.util.update.atomic.AtomicStatisti - I'm not sure if that means that it worked? I sent feedback to Atmire - Meeting with Moayad to discuss OpenRXV development progress +## 2021-04-25 + +- The indexes on AReS are messed up again + - I made a backup of the indexes, then deleted the `openrxv-items-final` and `openrxv-items-temp` indexes, re-created the `openrxv-items` alias, and restored the data into `openrxv-items`: + +```console +$ elasticdump --input=http://localhost:9200/openrxv-items --output=/home/aorth/openrxv-items_mapping.json --type=mapping +$ elasticdump --input=http://localhost:9200/openrxv-items --output=/home/aorth/openrxv-items_data.json --limit=1000 --type=data +$ curl -XDELETE 'http://localhost:9200/openrxv-items-temp' +$ curl -XDELETE 'http://localhost:9200/openrxv-items-final' +$ elasticdump --input=/home/aorth/openrxv-items_mapping.json --output=http://localhost:9200/openrxv-items-final --type=mapping +$ curl -s -X POST 'http://localhost:9200/_aliases' -H 'Content-Type: application/json' -d'{"actions" : [{"add" : { "index" : "openrxv-items-final", "alias" : "openrxv-items"}}]}' +$ elasticdump --input=/home/aorth/openrxv-items_data.json --output=http://localhost:9200/openrxv-items --limit 1000 --type=data +``` + +- Then I started a fresh AReS harvest + +## 2021-04-26 + +- The AReS harvest last night seems to have finished successfully and the number of items looks good: + +```console +$ curl -s http://localhost:9200/_cat/indices | grep openrxv-items +yellow open openrxv-items-temp H-CGsyyLTaqAj6-nKXZ-7w 1 1 0 0 283b 283b +yellow open openrxv-items-final ul3SKsa7Q9Cd_K7qokBY_w 1 1 103951 0 254mb 254mb +``` + +- And the aliases seem correct for once: + +```console +$ curl -s 'http://localhost:9200/_alias/' | python -m json.tool +... + "openrxv-items-final": { + "aliases": { + "openrxv-items": {} + } + }, + "openrxv-items-temp": { + "aliases": {} + }, +... +``` + +- That's 250 new items in the index since the last harvest! +- Re-create my local Artifactory container because I'm getting errors starting it and it has been a few months since it was updated: + +```console +$ podman rm artifactory +$ podman pull docker.bintray.io/jfrog/artifactory-oss:latest +$ podman create --ulimit nofile=32000:32000 --name artifactory -v artifactory_data:/var/opt/jfrog/artifactory -p 8081-8082:8081-8082 docker.bintray.io/jfrog/artifactory-oss +$ podman start artifactory +``` + +- Start testing DSpace 7.0 Beta 5 so I can evaluate if it solves some of the problems we are having on DSpace 6, and if it's missing things like multiple handle resolvers, etc + - I see it needs Java JDK 11, Tomcat 9, Solr 8, and PostgreSQL 11 + - Also, according to the [installation notes](https://wiki.lyrasis.org/display/DSDOC7x/Installing+DSpace) I see you can install the old DSpace 6 REST API, so that's potentially useful for us + - I see that all web applications on the backend are now rolled into just one "server" application + - The build process took 11 minutes the first time (due to downloading the world with Maven) and ~2 minutes the second time + - The `local.cfg` content and syntax is very similar DSpace 6 +- I got the basic `fresh_install` up and running + - Then I tried to import a DSpace 6 database from production +- I tried to delete all the Atmire SQL migrations: + +```console +localhost/dspace7b5= > DELETE FROM schema_version WHERE description LIKE '%Atmire%' OR description LIKE '%CUA%' OR description LIKE '%cua%'; +``` + +- But I got an error when running `dspace database migrate`: + +```console +$ ~/dspace7b5/bin/dspace database migrate + +Database URL: jdbc:postgresql://localhost:5432/dspace7b5 +Migrating database to latest version... (Check dspace logs for details) +Migration exception: +java.sql.SQLException: Flyway migration error occurred + at org.dspace.storage.rdbms.DatabaseUtils.updateDatabase(DatabaseUtils.java:738) + at org.dspace.storage.rdbms.DatabaseUtils.updateDatabase(DatabaseUtils.java:632) + at org.dspace.storage.rdbms.DatabaseUtils.main(DatabaseUtils.java:228) + at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.base/java.lang.reflect.Method.invoke(Method.java:566) + at org.dspace.app.launcher.ScriptLauncher.runOneCommand(ScriptLauncher.java:273) + at org.dspace.app.launcher.ScriptLauncher.handleScript(ScriptLauncher.java:129) + at org.dspace.app.launcher.ScriptLauncher.main(ScriptLauncher.java:94) +Caused by: org.flywaydb.core.api.FlywayException: Validate failed: +Detected applied migration not resolved locally: 5.0.2017.09.25 +Detected applied migration not resolved locally: 6.0.2017.01.30 +Detected applied migration not resolved locally: 6.0.2017.09.25 + + at org.flywaydb.core.Flyway.doValidate(Flyway.java:292) + at org.flywaydb.core.Flyway.access$100(Flyway.java:73) + at org.flywaydb.core.Flyway$1.execute(Flyway.java:166) + at org.flywaydb.core.Flyway$1.execute(Flyway.java:158) + at org.flywaydb.core.Flyway.execute(Flyway.java:527) + at org.flywaydb.core.Flyway.migrate(Flyway.java:158) + at org.dspace.storage.rdbms.DatabaseUtils.updateDatabase(DatabaseUtils.java:729) + ... 9 more +``` + +- I deleted those migrations: + +```console +localhost/dspace7b5= > DELETE FROM schema_version WHERE version IN ('5.0.2017.09.25', '6.0.2017.01.30', '6.0.2017.09.25'); +``` + +- Then when I ran the migration again it failed for a new reason, related to the configurable workflow: + +```console +Database URL: jdbc:postgresql://localhost:5432/dspace7b5 +Migrating database to latest version... (Check dspace logs for details) +Migration exception: +java.sql.SQLException: Flyway migration error occurred + at org.dspace.storage.rdbms.DatabaseUtils.updateDatabase(DatabaseUtils.java:738) + at org.dspace.storage.rdbms.DatabaseUtils.updateDatabase(DatabaseUtils.java:632) + at org.dspace.storage.rdbms.DatabaseUtils.main(DatabaseUtils.java:228) + at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.base/java.lang.reflect.Method.invoke(Method.java:566) + at org.dspace.app.launcher.ScriptLauncher.runOneCommand(ScriptLauncher.java:273) + at org.dspace.app.launcher.ScriptLauncher.handleScript(ScriptLauncher.java:129) + at org.dspace.app.launcher.ScriptLauncher.main(ScriptLauncher.java:94) +Caused by: org.flywaydb.core.internal.command.DbMigrate$FlywayMigrateException: +Migration V7.0_2019.05.02__DS-4239-workflow-xml-migration.sql failed +-------------------------------------------------------------------- +SQL State : 42P01 +Error Code : 0 +Message : ERROR: relation "cwf_pooltask" does not exist + Position: 8 +Location : org/dspace/storage/rdbms/sqlmigration/postgres/V7.0_2019.05.02__DS-4239-workflow-xml-migration.sql (/home/aorth/src/apache-tomcat-9.0.45/file:/home/aorth/dspace7b5/lib/dspace-api-7.0-beta5.jar!/org/dspace/storage/rdbms/sqlmigration/postgres/V7.0_2019.05.02__DS-4239-workflow-xml-migration.sql) +Line : 16 +Statement : UPDATE cwf_pooltask SET workflow_id='defaultWorkflow' WHERE workflow_id='default' +... +``` + +- The [DSpace 7 upgrade docs](https://wiki.lyrasis.org/display/DSDOC7x/Upgrading+DSpace) say I need to apply these previously optional migrations: + +```console +$ ~/dspace7b5/bin/dspace database migrate ignored +``` + +- Now I see all migrations have completed and DSpace actually starts up fine! +- I will try to do a full re-index to see how long it takes: + +```console +$ time ~/dspace7b5/bin/dspace index-discovery -b +``` + diff --git a/docs/2021-04/index.html b/docs/2021-04/index.html index 67bbd628b..946afacf8 100644 --- a/docs/2021-04/index.html +++ b/docs/2021-04/index.html @@ -24,7 +24,7 @@ Perhaps one of the containers crashed, I should have looked closer but I was in - + @@ -54,9 +54,9 @@ Perhaps one of the containers crashed, I should have looked closer but I was in "@type": "BlogPosting", "headline": "April, 2021", "url": "https://alanorth.github.io/cgspace-notes/2021-04/", - "wordCount": "3896", + "wordCount": "4547", "datePublished": "2021-04-01T09:50:54+03:00", - "dateModified": "2021-04-19T20:11:44+03:00", + "dateModified": "2021-04-21T22:53:24+03:00", "author": { "@type": "Person", "name": "Alan Orth" @@ -861,7 +861,145 @@ $ chrt -b 0 dspace dsrun com.atmire.statistics.util.update.atomic.AtomicStatisti
  • Meeting with Moayad to discuss OpenRXV development progress
    • - +

    2021-04-25

    + +
    $ elasticdump --input=http://localhost:9200/openrxv-items --output=/home/aorth/openrxv-items_mapping.json --type=mapping
+$ elasticdump --input=http://localhost:9200/openrxv-items --output=/home/aorth/openrxv-items_data.json --limit=1000 --type=data
+$ curl -XDELETE 'http://localhost:9200/openrxv-items-temp'
+$ curl -XDELETE 'http://localhost:9200/openrxv-items-final'
+$ elasticdump --input=/home/aorth/openrxv-items_mapping.json --output=http://localhost:9200/openrxv-items-final --type=mapping
+$ curl -s -X POST 'http://localhost:9200/_aliases' -H 'Content-Type: application/json' -d'{"actions" : [{"add" : { "index" : "openrxv-items-final", "alias" : "openrxv-items"}}]}'
+$ elasticdump --input=/home/aorth/openrxv-items_data.json --output=http://localhost:9200/openrxv-items --limit 1000 --type=data
+
    +

    2021-04-26

    + +
    $ curl -s http://localhost:9200/_cat/indices | grep openrxv-items
+yellow open openrxv-items-temp       H-CGsyyLTaqAj6-nKXZ-7w 1 1      0 0    283b    283b
+yellow open openrxv-items-final      ul3SKsa7Q9Cd_K7qokBY_w 1 1 103951 0   254mb   254mb
+
    +
    $ curl -s 'http://localhost:9200/_alias/' | python -m json.tool
+...
+    "openrxv-items-final": {
+        "aliases": {
+            "openrxv-items": {}
+        }
+    },
+    "openrxv-items-temp": {
+        "aliases": {}
+    },
+...
+
    +
    $ podman rm artifactory
+$ podman pull docker.bintray.io/jfrog/artifactory-oss:latest
+$ podman create --ulimit nofile=32000:32000 --name artifactory -v artifactory_data:/var/opt/jfrog/artifactory -p 8081-8082:8081-8082 docker.bintray.io/jfrog/artifactory-oss
+$ podman start artifactory
+
    +
    localhost/dspace7b5= > DELETE FROM schema_version WHERE description LIKE '%Atmire%' OR description LIKE '%CUA%' OR description LIKE '%cua%';
+
    +
    $ ~/dspace7b5/bin/dspace database migrate
+
+Database URL: jdbc:postgresql://localhost:5432/dspace7b5
+Migrating database to latest version... (Check dspace logs for details)
+Migration exception:
+java.sql.SQLException: Flyway migration error occurred
+        at org.dspace.storage.rdbms.DatabaseUtils.updateDatabase(DatabaseUtils.java:738)
+        at org.dspace.storage.rdbms.DatabaseUtils.updateDatabase(DatabaseUtils.java:632)
+        at org.dspace.storage.rdbms.DatabaseUtils.main(DatabaseUtils.java:228)
+        at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
+        at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
+        at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
+        at java.base/java.lang.reflect.Method.invoke(Method.java:566)
+        at org.dspace.app.launcher.ScriptLauncher.runOneCommand(ScriptLauncher.java:273)
+        at org.dspace.app.launcher.ScriptLauncher.handleScript(ScriptLauncher.java:129)
+        at org.dspace.app.launcher.ScriptLauncher.main(ScriptLauncher.java:94)
+Caused by: org.flywaydb.core.api.FlywayException: Validate failed: 
+Detected applied migration not resolved locally: 5.0.2017.09.25
+Detected applied migration not resolved locally: 6.0.2017.01.30
+Detected applied migration not resolved locally: 6.0.2017.09.25
+
+        at org.flywaydb.core.Flyway.doValidate(Flyway.java:292)
+        at org.flywaydb.core.Flyway.access$100(Flyway.java:73)
+        at org.flywaydb.core.Flyway$1.execute(Flyway.java:166)
+        at org.flywaydb.core.Flyway$1.execute(Flyway.java:158)
+        at org.flywaydb.core.Flyway.execute(Flyway.java:527)
+        at org.flywaydb.core.Flyway.migrate(Flyway.java:158)
+        at org.dspace.storage.rdbms.DatabaseUtils.updateDatabase(DatabaseUtils.java:729)
+        ... 9 more
+
    +
    localhost/dspace7b5= > DELETE FROM schema_version WHERE version IN ('5.0.2017.09.25', '6.0.2017.01.30', '6.0.2017.09.25');
+
    +
    Database URL: jdbc:postgresql://localhost:5432/dspace7b5
+Migrating database to latest version... (Check dspace logs for details)
+Migration exception:
+java.sql.SQLException: Flyway migration error occurred
+        at org.dspace.storage.rdbms.DatabaseUtils.updateDatabase(DatabaseUtils.java:738)
+        at org.dspace.storage.rdbms.DatabaseUtils.updateDatabase(DatabaseUtils.java:632)
+        at org.dspace.storage.rdbms.DatabaseUtils.main(DatabaseUtils.java:228)
+        at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
+        at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
+        at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
+        at java.base/java.lang.reflect.Method.invoke(Method.java:566)
+        at org.dspace.app.launcher.ScriptLauncher.runOneCommand(ScriptLauncher.java:273)
+        at org.dspace.app.launcher.ScriptLauncher.handleScript(ScriptLauncher.java:129)
+        at org.dspace.app.launcher.ScriptLauncher.main(ScriptLauncher.java:94)
+Caused by: org.flywaydb.core.internal.command.DbMigrate$FlywayMigrateException:
+Migration V7.0_2019.05.02__DS-4239-workflow-xml-migration.sql failed
+--------------------------------------------------------------------
+SQL State  : 42P01
+Error Code : 0
+Message    : ERROR: relation "cwf_pooltask" does not exist
+  Position: 8
+Location   : org/dspace/storage/rdbms/sqlmigration/postgres/V7.0_2019.05.02__DS-4239-workflow-xml-migration.sql (/home/aorth/src/apache-tomcat-9.0.45/file:/home/aorth/dspace7b5/lib/dspace-api-7.0-beta5.jar!/org/dspace/storage/rdbms/sqlmigration/postgres/V7.0_2019.05.02__DS-4239-workflow-xml-migration.sql)
+Line       : 16
+Statement  : UPDATE cwf_pooltask SET workflow_id='defaultWorkflow' WHERE workflow_id='default'
+...
+
    +
    $ ~/dspace7b5/bin/dspace database migrate ignored
+
    +
    $ time ~/dspace7b5/bin/dspace index-discovery -b
+
    diff --git a/docs/categories/index.html b/docs/categories/index.html index 8bdd00864..e22c23f85 100644 --- a/docs/categories/index.html +++ b/docs/categories/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/categories/notes/index.html b/docs/categories/notes/index.html index 37c1ee1ed..eb170b49b 100644 --- a/docs/categories/notes/index.html +++ b/docs/categories/notes/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/categories/notes/page/2/index.html b/docs/categories/notes/page/2/index.html index 5af8f6205..3e6f1c4fb 100644 --- a/docs/categories/notes/page/2/index.html +++ b/docs/categories/notes/page/2/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/categories/notes/page/3/index.html b/docs/categories/notes/page/3/index.html index 86ca8a808..e9045a339 100644 --- a/docs/categories/notes/page/3/index.html +++ b/docs/categories/notes/page/3/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/categories/notes/page/4/index.html b/docs/categories/notes/page/4/index.html index 9b1cc4679..a90feb906 100644 --- a/docs/categories/notes/page/4/index.html +++ b/docs/categories/notes/page/4/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/categories/notes/page/5/index.html b/docs/categories/notes/page/5/index.html index 3005bd0d3..a925f47d2 100644 --- a/docs/categories/notes/page/5/index.html +++ b/docs/categories/notes/page/5/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/index.html b/docs/index.html index 5981d9022..1ece27357 100644 --- a/docs/index.html +++ b/docs/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/page/2/index.html b/docs/page/2/index.html index e4b54eaef..6e7b9e12d 100644 --- a/docs/page/2/index.html +++ b/docs/page/2/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/page/3/index.html b/docs/page/3/index.html index 2a0beb4f9..749cfca7a 100644 --- a/docs/page/3/index.html +++ b/docs/page/3/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/page/4/index.html b/docs/page/4/index.html index f0bdbd645..ec7bc62e4 100644 --- a/docs/page/4/index.html +++ b/docs/page/4/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/page/5/index.html b/docs/page/5/index.html index c63a121c3..071ccbeeb 100644 --- a/docs/page/5/index.html +++ b/docs/page/5/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/page/6/index.html b/docs/page/6/index.html index 7ebfc4f54..b8e8c5521 100644 --- a/docs/page/6/index.html +++ b/docs/page/6/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/page/7/index.html b/docs/page/7/index.html index 331a7d8a2..d980801ea 100644 --- a/docs/page/7/index.html +++ b/docs/page/7/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/posts/index.html b/docs/posts/index.html index e02855806..9df4092ed 100644 --- a/docs/posts/index.html +++ b/docs/posts/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/posts/page/2/index.html b/docs/posts/page/2/index.html index 6fe55dfa3..09ee131d4 100644 --- a/docs/posts/page/2/index.html +++ b/docs/posts/page/2/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/posts/page/3/index.html b/docs/posts/page/3/index.html index 31301d71c..7d03b4e4f 100644 --- a/docs/posts/page/3/index.html +++ b/docs/posts/page/3/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/posts/page/4/index.html b/docs/posts/page/4/index.html index fe562f288..c77accff3 100644 --- a/docs/posts/page/4/index.html +++ b/docs/posts/page/4/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/posts/page/5/index.html b/docs/posts/page/5/index.html index 5581c1230..2c17bd0d3 100644 --- a/docs/posts/page/5/index.html +++ b/docs/posts/page/5/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/posts/page/6/index.html b/docs/posts/page/6/index.html index 7ed606127..b00ca84cb 100644 --- a/docs/posts/page/6/index.html +++ b/docs/posts/page/6/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/posts/page/7/index.html b/docs/posts/page/7/index.html index fe5ff855e..00829ef59 100644 --- a/docs/posts/page/7/index.html +++ b/docs/posts/page/7/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/sitemap.xml b/docs/sitemap.xml index 2f0348b8b..584b79cef 100644 --- a/docs/sitemap.xml +++ b/docs/sitemap.xml @@ -3,19 +3,19 @@ xmlns:xhtml="http://www.w3.org/1999/xhtml"> https://alanorth.github.io/cgspace-notes/2021-04/ - 2021-04-19T20:11:44+03:00 + 2021-04-21T22:53:24+03:00 https://alanorth.github.io/cgspace-notes/categories/ - 2021-04-19T20:11:44+03:00 + 2021-04-21T22:53:24+03:00 https://alanorth.github.io/cgspace-notes/ - 2021-04-19T20:11:44+03:00 + 2021-04-21T22:53:24+03:00 https://alanorth.github.io/cgspace-notes/categories/notes/ - 2021-04-19T20:11:44+03:00 + 2021-04-21T22:53:24+03:00 https://alanorth.github.io/cgspace-notes/posts/ - 2021-04-19T20:11:44+03:00 + 2021-04-21T22:53:24+03:00 https://alanorth.github.io/cgspace-notes/2021-03/ 2021-04-13T21:13:08+03:00