From 9d32f8e270a6b2ab542c0b06c01aa028e0f43ba3 Mon Sep 17 00:00:00 2001 From: Alan Orth Date: Mon, 8 Feb 2016 08:59:05 +0200 Subject: [PATCH] Update notes for 2016-02-07 Signed-off-by: Alan Orth --- content/2016-02.md | 18 +- public/2015-11/index.html | 163 ++++++++++++------ public/2015-12/index.html | 163 ++++++++++++------ public/2016-01/index.html | 163 ++++++++++++------ public/2016-02/index.html | 182 +++++++++++++++------ public/404.html | 124 +++++++++----- public/index.html | 308 ++++++++++++++++++++++------------- public/index.xml | 19 ++- public/tags/notes/index.html | 268 ++++++++++++++++++++---------- public/tags/notes/index.xml | 19 ++- 10 files changed, 994 insertions(+), 433 deletions(-) diff --git a/content/2016-02.md b/content/2016-02.md index b32f9c409..93c19cdab 100644 --- a/content/2016-02.md +++ b/content/2016-02.md @@ -59,12 +59,22 @@ DELETE 25 - For some reason when you import an Excel file into OpenRefine it exports dates like 1949 to 1949.0 in the CSV - I re-import the resulting CSV and run a GREL on the date issued column: `value.replace("\.0", "")` - I need to start running DSpace in Mac OS X instead of a Linux VM -- Install PostgreSQL from homebrew and configure: +- Install PostgreSQL from homebrew, then configure and import CGSpace database dump: ``` $ postgres -D /opt/brew/var/postgres +$ createuser --superuser postgres $ createuser --pwprompt dspacetest $ createdb -O dspacetest --encoding=UNICODE dspacetest +$ psql postgres +postgres=# alter user dspacetest createuser; +postgres=# \q +$ pg_restore -O -U dspacetest -d dspacetest ~/Downloads/cgspace_2016-02-07.backup +$ psql postgres +postgres=# alter user dspacetest nocreateuser; +postgres=# \q +$ vacuumdb dspacetest +$ psql -U dspacetest -f ~/src/git/DSpace/dspace/etc/postgres/update-sequences.sql dspacetest -h localhost ``` - After building and running a `fresh_install` I symlinked the webapps into Tomcat's webapps folder: @@ -85,3 +95,9 @@ $ /opt/brew/Cellar/tomcat/8.0.30/bin/catalina start ``` CATALINA_OPTS="-Djava.awt.headless=true -Xms2048m -Xmx2048m -XX:MaxPermSize=256m -XX:+UseConcMarkSweepGC -Dfile.encoding=UTF-8" ``` + +- After verifying that the site is working, start a full index: + +``` +$ ~/dspace/bin/dspace index-discovery -b +``` diff --git a/public/2015-11/index.html b/public/2015-11/index.html index 55fe687f8..5f661fbd3 100644 --- a/public/2015-11/index.html +++ b/public/2015-11/index.html @@ -1,50 +1,82 @@ - - - - - - - - - - -November, 2015 - CGSpace Notes + + + + + + + + + + + + + + + + + + + November, 2015 · CGSpace Notes + + + + + + + + + + - -
-
- [CGSpace Notes] - # _ -
-
- -
- - -
+
+
+

CGSpace Notes

+ +
-
-
+ + + + + + + + +
- published on - + + + + +
+
+
- -
- tags: - - - notes - - - -
-

November, 2015

-
+
+
+

November, 2015

+
+
+ Posted on + +
+
+ + #notes + +
+
+
+

2015-11-22

@@ -193,19 +225,56 @@ db.statementpool = true
+
- - - + +
+
+ +
+
+ + + + + + + + +
- +
+ + + + diff --git a/public/2015-12/index.html b/public/2015-12/index.html index 7740ff204..3266379a2 100644 --- a/public/2015-12/index.html +++ b/public/2015-12/index.html @@ -1,50 +1,82 @@ - - - - - - - - - - -December, 2015 - CGSpace Notes + + + + + + + + + + + + + + + + + + + December, 2015 · CGSpace Notes + + + + + + + + + + - -
-
- [CGSpace Notes] - # _ -
-
- -
- - -
+
+
+

CGSpace Notes

+ +
-
-
+ + + + + + + + +
- published on - + + + + +
+
+
- -
- tags: - - - notes - - - -
-

December, 2015

-
+
+
+

December, 2015

+
+
+ Posted on + +
+
+ + #notes + +
+
+
+

2015-12-02

@@ -208,19 +240,56 @@ $ curl -o /dev/null -s -w %{time_total}\\n https://cgspace.cgiar.org/rest/handle
+
- - - + +
+
+ +
+
+ + + + + + + + +
- +
+ + + + diff --git a/public/2016-01/index.html b/public/2016-01/index.html index db6865f18..fe923194d 100644 --- a/public/2016-01/index.html +++ b/public/2016-01/index.html @@ -1,50 +1,82 @@ - - - - - - - - - - -January, 2016 - CGSpace Notes + + + + + + + + + + + + + + + + + + + January, 2016 · CGSpace Notes + + + + + + + + + + - -
-
- [CGSpace Notes] - # _ -
-
- -
- - -
+
+
+

CGSpace Notes

+ +
-
-
+ + + + + + + + +
- published on - + + + + +
+
+
- -
- tags: - - - notes - - - -
-

January, 2016

-
+
+
+

January, 2016

+
+
+ Posted on + +
+
+ + #notes + +
+
+
+

2016-01-13

@@ -133,19 +165,56 @@ $ find SimpleArchiveForBio/ -iname “*.pdf” -exec basename {} \; | so

XMLUI subjects after

+
- - - + +
+
+ +
+
+ + + + + + + + +
- +
+ + + + diff --git a/public/2016-02/index.html b/public/2016-02/index.html index 22f92eafb..a5eaf7c81 100644 --- a/public/2016-02/index.html +++ b/public/2016-02/index.html @@ -1,50 +1,82 @@ - - - - - - - - - - -February, 2016 - CGSpace Notes + + + + + + + + + + + + + + + + + + + February, 2016 · CGSpace Notes + + + + + + + + + + - -
-
- [CGSpace Notes] - # _ -
-
- -
- - -
+
+
+

CGSpace Notes

+ +
-
-
+ + + + + + + + +
- published on - + + + + +
+
+
- -
- tags: - - - notes - - - -
-

February, 2016

-
+
+
+

February, 2016

+
+
+ Posted on + +
+
+ + #notes + +
+
+
+

2016-02-05

@@ -111,12 +143,22 @@ DELETE 25
  • For some reason when you import an Excel file into OpenRefine it exports dates like 1949 to 1949.0 in the CSV
  • I re-import the resulting CSV and run a GREL on the date issued column: value.replace("\.0", "")
  • I need to start running DSpace in Mac OS X instead of a Linux VM
    • -
  • Install PostgreSQL from homebrew and configure:
    • +
  • Install PostgreSQL from homebrew, then configure and import CGSpace database dump:
    • $ postgres -D /opt/brew/var/postgres
+$ createuser --superuser postgres
 $ createuser --pwprompt dspacetest
 $ createdb -O dspacetest --encoding=UNICODE dspacetest
+$ psql postgres
+postgres=# alter user dspacetest createuser;
+postgres=# \q
+$ pg_restore -O -U dspacetest -d dspacetest ~/Downloads/cgspace_2016-02-07.backup 
+$ psql postgres
+postgres=# alter user dspacetest nocreateuser;
+postgres=# \q
+$ vacuumdb dspacetest
+$ psql -U dspacetest -f ~/src/git/DSpace/dspace/etc/postgres/update-sequences.sql dspacetest -h localhost
      @@ -138,22 +180,66 @@ $ /opt/brew/Cellar/tomcat/8.0.30/bin/catalina start 
    CATALINA_OPTS="-Djava.awt.headless=true -Xms2048m -Xmx2048m -XX:MaxPermSize=256m -XX:+UseConcMarkSweepGC -Dfile.encoding=UTF-8"
+
    + +
      +
    • After verifying that the site is working, start a full index:
      • +
    + +
    $ ~/dspace/bin/dspace index-discovery -b
    +
    - - - + +
    +
    + +
    +
    + + + + + + + + +
    - +
    + + + + diff --git a/public/404.html b/public/404.html index 8ffb8cb96..5bf3f8568 100644 --- a/public/404.html +++ b/public/404.html @@ -1,51 +1,93 @@ - - - - - - - - - - -404 Page not found - CGSpace Notes + + + + + + + + + + + + + + 404 Page not found · CGSpace Notes + + + + + + + + + + - -
    -
    - [CGSpace Notes] - # _ -
    -
    - -
    - - -
    - -

    - [404] - # _ -

    - -

    - not found -

    - -
    - - +
    +
    +

    CGSpace Notes

    + +
    + + + + + + + + +
    - + + + + + + +
    +
    +
    +
    +

    404

    +
    That page doesn't exist!
    +
    +
    +
    +
    + +
    +
    + + + + + + + + + +
    + +
    + + + + + + diff --git a/public/index.html b/public/index.html index 3715e87d8..25ae8d191 100644 --- a/public/index.html +++ b/public/index.html @@ -1,126 +1,206 @@ - - - - - - - - - - -CGSpace Notes + + + + + + + + + + + + + + CGSpace Notes + + + + + + + + + + + + - -
    -
    - [CGSpace Notes] - # _ -
    -
    - -
    - - - -
    - -

    - [base16] - # _ -

    - -
    - - - -
    - - - + + + + + diff --git a/public/index.xml b/public/index.xml index 99ef90a1a..da8e8e357 100644 --- a/public/index.xml +++ b/public/index.xml @@ -81,12 +81,22 @@ DELETE 25 <li>For some reason when you import an Excel file into OpenRefine it exports dates like 1949 to 1949.0 in the CSV</li> <li>I re-import the resulting CSV and run a GREL on the date issued column: <code>value.replace(&quot;\.0&quot;, &quot;&quot;)</code></li> <li>I need to start running DSpace in Mac OS X instead of a Linux VM</li> -<li>Install PostgreSQL from homebrew and configure:</li> +<li>Install PostgreSQL from homebrew, then configure and import CGSpace database dump:</li> </ul> <pre><code>$ postgres -D /opt/brew/var/postgres +$ createuser --superuser postgres $ createuser --pwprompt dspacetest $ createdb -O dspacetest --encoding=UNICODE dspacetest +$ psql postgres +postgres=# alter user dspacetest createuser; +postgres=# \q +$ pg_restore -O -U dspacetest -d dspacetest ~/Downloads/cgspace_2016-02-07.backup +$ psql postgres +postgres=# alter user dspacetest nocreateuser; +postgres=# \q +$ vacuumdb dspacetest +$ psql -U dspacetest -f ~/src/git/DSpace/dspace/etc/postgres/update-sequences.sql dspacetest -h localhost </code></pre> <ul> @@ -109,6 +119,13 @@ $ /opt/brew/Cellar/tomcat/8.0.30/bin/catalina start <pre><code>CATALINA_OPTS=&quot;-Djava.awt.headless=true -Xms2048m -Xmx2048m -XX:MaxPermSize=256m -XX:+UseConcMarkSweepGC -Dfile.encoding=UTF-8&quot; </code></pre> + +<ul> +<li>After verifying that the site is working, start a full index:</li> +</ul> + +<pre><code>$ ~/dspace/bin/dspace index-discovery -b +</code></pre> diff --git a/public/tags/notes/index.html b/public/tags/notes/index.html index c9a86059a..057dbd441 100644 --- a/public/tags/notes/index.html +++ b/public/tags/notes/index.html @@ -1,101 +1,197 @@ - - - - - - - - - - -Notes - CGSpace Notes + + + + + + + + + + + + + + Notes · CGSpace Notes + + + + + + + + + + - -
    -
    - [CGSpace Notes] - # _ -
    -
    - -
    - -
    -

    Notes

    +
    +
    +

    CGSpace Notes

    + +
    + + + + - -
    -

    February, 2016

    -
    - - published on - - -
    -
    - 2016-02-05 Looking at some DAGRIS data for Abenet Yabowork Lots of issues with spaces, newlines, etc causing the import to fail I noticed we have a very interesting list of countries on CGSpace: Not only are there 49,000 countries, we have some blanks (25)… Also, lots of things like “COTE D`LVOIRE” and “COTE D IVOIRE” 2016-02-06 Found a way to get items with null/empty metadata values from SQL First, find the metadata_field_id for the field you want from the metadatafieldregistry table: dspacetest=# select * from metadatafieldregistry; In this case our country field is 78 Now find all resources with type 2 (item) that have null/empty values for that field: dspacetest=# select resource_id from metadatavalue where resource_type_id=2 and metadata_field_id=78 and (text_value='' OR text_value IS NULL); Then you can find the handle that owns it from its resource_id: dspacetest=# select handle from item, handle where handle.resource_id = item.item_id AND item.item_id = '22678'; It’s 25 items so editing in the web UI is annoying, let’s try SQL! Read More... -
    -
    - - -
    -

    January, 2016

    -
    - - published on - - -
    -
    - 2016-01-13 Move ILRI collection 10568/12503 from 10568/27869 to 10568/27629 using the move_collections.sh script I wrote last year. I realized it is only necessary to clear the Cocoon cache after moving collections—rather than reindexing—as no metadata has changed, and therefore no search or browse indexes need to be updated. Update GitHub wiki for documentation of maintenance tasks. 2016-01-14 Update CCAFS project identifiers in input-forms.xml Run system updates and restart the server 2016-01-18 Change “Extension material” to “Extension Material” in input-forms.xml (a mistake that fell through the cracks when we fixed the others in DSpace 4 era) 2016-01-19 Work on tweaks and updates for the social sharing icons on item pages: add Delicious and Mendeley (from Academicons), make links open in new windows, and set the icon color to the theme’s primary color (#157) Tweak date-based facets to show more values in drill-down ranges (#162) Need to remember to clear the Cocoon cache after deployment or else you don’t see the new ranges immediately Set up recipe on IFTTT to tweet new items from the CGSpace Atom feed to my twitter account Altmetrics’ support for Handles is kinda weak, so they can’t associate our items with DOIs until they are tweeted or blogged, etc first. Read More... -
    -
    - - -
    -

    December, 2015

    -
    - - published on - - -
    -
    - 2015-12-02 Replace lzop with xz in log compression cron jobs on DSpace Test—it uses less space: # cd /home/dspacetest.cgiar.org/log # ls -lh dspace.log.2015-11-18* -rw-rw-r-- 1 tomcat7 tomcat7 2.0M Nov 18 23:59 dspace.log.2015-11-18 -rw-rw-r-- 1 tomcat7 tomcat7 387K Nov 18 23:59 dspace.log.2015-11-18.lzo -rw-rw-r-- 1 tomcat7 tomcat7 169K Nov 18 23:59 dspace.log.2015-11-18.xz I had used lrzip once, but it needs more memory and is harder to use as it requires the lrztar Read More... -
    -
    - - -
    -

    November, 2015

    -
    - - published on - - -
    -
    - 2015-11-22 CGSpace went down Looks like DSpace exhausted its PostgreSQL connection pool Last week I had increased the limit from 30 to 60, which seemed to help, but now there are many more idle connections: $ psql -c 'SELECT * from pg_stat_activity;' | grep idle | grep -c cgspace 78 For now I have increased the limit from 60 to 90, run updates, and rebooted the server 2015-11-24 CGSpace went down again Getting emails from uptimeRobot and uptimeButler that it’s down, and Google Webmaster Tools is sending emails that there is an increase in crawl errors Looks like there are still a bunch of idle PostgreSQL connections: $ psql -c 'SELECT * from pg_stat_activity;' | grep idle | grep -c cgspace 96 For some reason the number of idle connections is very high since we upgraded to DSpace 5 2015-11-25 Troubleshoot the DSpace 5 OAI breakage caused by nginx routing config The OAI application requests stylesheets and javascript files with the path /oai/static/css, which gets matched here: # static assets we can load from the file system directly with nginx location ~ /(themes|static|aspects/ReportingSuite) { try_files $uri @tomcat; ... Read More... -
    -
    - -
    -
    - + + + + + + +
    + +
    +
    +

    Notes

    + +
    +
    +
    +

    February, 2016

    +
    +
    + Posted on + +
    +
    +
    +
    + 2016-02-05 Looking at some DAGRIS data for Abenet Yabowork Lots of issues with spaces, newlines, etc causing the import to fail I noticed we have a very interesting list of countries on CGSpace: Not only are there 49,000 countries, we have some blanks (25)… Also, lots of things like “COTE D`LVOIRE” and “COTE D IVOIRE” 2016-02-06 Found a way to get items with null/empty metadata values from SQL First, find the metadata_field_id for the field you want from the metadatafieldregistry table: dspacetest=# select * from metadatafieldregistry; In this case our country field is 78 Now find all resources with type 2 (item) that have null/empty values for that field: dspacetest=# select resource_id from metadatavalue where resource_type_id=2 and metadata_field_id=78 and (text_value='' OR text_value IS NULL); Then you can find the handle that owns it from its resource_id: dspacetest=# select handle from item, handle where handle.resource_id = item.item_id AND item.item_id = '22678'; It’s 25 items so editing in the web UI is annoying, let’s try SQL! +
    + + + +
    + + +
    +
    +
    +

    January, 2016

    +
    +
    + Posted on + +
    +
    +
    +
    + 2016-01-13 Move ILRI collection 10568/12503 from 10568/27869 to 10568/27629 using the move_collections.sh script I wrote last year. I realized it is only necessary to clear the Cocoon cache after moving collections—rather than reindexing—as no metadata has changed, and therefore no search or browse indexes need to be updated. Update GitHub wiki for documentation of maintenance tasks. 2016-01-14 Update CCAFS project identifiers in input-forms.xml Run system updates and restart the server 2016-01-18 Change “Extension material” to “Extension Material” in input-forms.xml (a mistake that fell through the cracks when we fixed the others in DSpace 4 era) 2016-01-19 Work on tweaks and updates for the social sharing icons on item pages: add Delicious and Mendeley (from Academicons), make links open in new windows, and set the icon color to the theme’s primary color (#157) Tweak date-based facets to show more values in drill-down ranges (#162) Need to remember to clear the Cocoon cache after deployment or else you don’t see the new ranges immediately Set up recipe on IFTTT to tweet new items from the CGSpace Atom feed to my twitter account Altmetrics’ support for Handles is kinda weak, so they can’t associate our items with DOIs until they are tweeted or blogged, etc first. +
    + + + +
    + + +
    +
    +
    +

    December, 2015

    +
    +
    + Posted on + +
    +
    +
    +
    + 2015-12-02 Replace lzop with xz in log compression cron jobs on DSpace Test—it uses less space: # cd /home/dspacetest.cgiar.org/log # ls -lh dspace.log.2015-11-18* -rw-rw-r-- 1 tomcat7 tomcat7 2.0M Nov 18 23:59 dspace.log.2015-11-18 -rw-rw-r-- 1 tomcat7 tomcat7 387K Nov 18 23:59 dspace.log.2015-11-18.lzo -rw-rw-r-- 1 tomcat7 tomcat7 169K Nov 18 23:59 dspace.log.2015-11-18.xz I had used lrzip once, but it needs more memory and is harder to use as it requires the lrztar +
    + + + +
    + + +
    +
    +
    +

    November, 2015

    +
    +
    + Posted on + +
    +
    +
    +
    + 2015-11-22 CGSpace went down Looks like DSpace exhausted its PostgreSQL connection pool Last week I had increased the limit from 30 to 60, which seemed to help, but now there are many more idle connections: $ psql -c 'SELECT * from pg_stat_activity;' | grep idle | grep -c cgspace 78 For now I have increased the limit from 60 to 90, run updates, and rebooted the server 2015-11-24 CGSpace went down again Getting emails from uptimeRobot and uptimeButler that it’s down, and Google Webmaster Tools is sending emails that there is an increase in crawl errors Looks like there are still a bunch of idle PostgreSQL connections: $ psql -c 'SELECT * from pg_stat_activity;' | grep idle | grep -c cgspace 96 For some reason the number of idle connections is very high since we upgraded to DSpace 5 2015-11-25 Troubleshoot the DSpace 5 OAI breakage caused by nginx routing config The OAI application requests stylesheets and javascript files with the path /oai/static/css, which gets matched here: # static assets we can load from the file system directly with nginx location ~ /(themes|static|aspects/ReportingSuite) { try_files $uri @tomcat; ... +
    + + + +
    + + +
    +
    +
    +
    + +
    +
    + + + + + + + + + +
    + +
    + + + + + + diff --git a/public/tags/notes/index.xml b/public/tags/notes/index.xml index dd9a2933d..37ac38891 100644 --- a/public/tags/notes/index.xml +++ b/public/tags/notes/index.xml @@ -81,12 +81,22 @@ DELETE 25 <li>For some reason when you import an Excel file into OpenRefine it exports dates like 1949 to 1949.0 in the CSV</li> <li>I re-import the resulting CSV and run a GREL on the date issued column: <code>value.replace(&quot;\.0&quot;, &quot;&quot;)</code></li> <li>I need to start running DSpace in Mac OS X instead of a Linux VM</li> -<li>Install PostgreSQL from homebrew and configure:</li> +<li>Install PostgreSQL from homebrew, then configure and import CGSpace database dump:</li> </ul> <pre><code>$ postgres -D /opt/brew/var/postgres +$ createuser --superuser postgres $ createuser --pwprompt dspacetest $ createdb -O dspacetest --encoding=UNICODE dspacetest +$ psql postgres +postgres=# alter user dspacetest createuser; +postgres=# \q +$ pg_restore -O -U dspacetest -d dspacetest ~/Downloads/cgspace_2016-02-07.backup +$ psql postgres +postgres=# alter user dspacetest nocreateuser; +postgres=# \q +$ vacuumdb dspacetest +$ psql -U dspacetest -f ~/src/git/DSpace/dspace/etc/postgres/update-sequences.sql dspacetest -h localhost </code></pre> <ul> @@ -109,6 +119,13 @@ $ /opt/brew/Cellar/tomcat/8.0.30/bin/catalina start <pre><code>CATALINA_OPTS=&quot;-Djava.awt.headless=true -Xms2048m -Xmx2048m -XX:MaxPermSize=256m -XX:+UseConcMarkSweepGC -Dfile.encoding=UTF-8&quot; </code></pre> + +<ul> +<li>After verifying that the site is working, start a full index:</li> +</ul> + +<pre><code>$ ~/dspace/bin/dspace index-discovery -b +</code></pre>