From fa7df494c6675d76c2abfc16e7bb5da0efe5128d Mon Sep 17 00:00:00 2001 From: Alan Orth Date: Fri, 1 Dec 2017 13:55:00 +0300 Subject: [PATCH] Add notes for 2017-12-01 --- content/post/2017-12.md | 79 +++++++ public/2015-11/index.html | 8 +- public/2015-12/index.html | 8 +- public/2016-01/index.html | 8 +- public/2016-02/index.html | 8 +- public/2016-03/index.html | 8 +- public/2016-04/index.html | 8 +- public/2016-05/index.html | 8 +- public/2016-06/index.html | 8 +- public/2016-07/index.html | 8 +- public/2016-08/index.html | 8 +- public/2016-09/index.html | 8 +- public/2016-10/index.html | 8 +- public/2016-11/index.html | 8 +- public/2016-12/index.html | 8 +- public/2017-01/index.html | 8 +- public/2017-02/index.html | 8 +- public/2017-03/index.html | 8 +- public/2017-04/index.html | 8 +- public/2017-05/index.html | 8 +- public/2017-06/index.html | 8 +- public/2017-07/index.html | 8 +- public/2017-08/index.html | 8 +- public/2017-09/index.html | 8 +- public/2017-10/index.html | 8 +- public/2017-11/index.html | 8 +- public/2017-12/index.html | 270 ++++++++++++++++++++++ public/categories/notes/index.html | 4 +- public/cgiar-library-migration/index.html | 8 +- public/index.html | 77 +++--- public/index.xml | 20 +- public/page/2/index.html | 80 ++++--- public/page/3/index.html | 38 ++- public/post/index.html | 77 +++--- public/post/index.xml | 20 +- public/post/page/2/index.html | 80 ++++--- public/post/page/3/index.html | 38 ++- public/robots.txt | 1 + public/sitemap.xml | 67 +++--- public/tags/index.xml | 4 +- public/tags/notes/index.html | 75 +++--- public/tags/notes/index.xml | 20 +- public/tags/notes/page/2/index.html | 76 +++--- public/tags/notes/page/3/index.html | 36 ++- 44 files changed, 880 insertions(+), 390 deletions(-) create mode 100644 content/post/2017-12.md create mode 100644 public/2017-12/index.html diff --git a/content/post/2017-12.md b/content/post/2017-12.md new file mode 100644 index 000000000..74d00ee72 --- /dev/null +++ b/content/post/2017-12.md @@ -0,0 +1,79 @@ +--- +title: "December, 2017" +date: 2017-12-01T13:53:54+03:00 +author: "Alan Orth" +tags: ["Notes"] +--- + +## 2017-12-01 + +- Uptime Robot noticed that CGSpace went down +- The logs say "Timeout waiting for idle object" +- PostgreSQL activity says there are 115 connections currently +- The list of connections to XMLUI and REST API for today: + + + +``` +# cat /var/log/nginx/rest.log /var/log/nginx/rest.log.1 /var/log/nginx/access.log /var/log/nginx/access.log.1 /var/log/nginx/library-access.log /var/log/nginx/library-access.log.1 | grep -E "1/Dec/2017" | awk '{print $1}' | sort -n | uniq -c | sort -h | tail + 763 2.86.122.76 + 907 207.46.13.94 + 1018 157.55.39.206 + 1021 157.55.39.235 + 1407 66.249.66.70 + 1411 104.196.152.243 + 1503 50.116.102.77 + 1805 66.249.66.90 + 4007 70.32.83.92 + 6061 45.5.184.196 +``` + +- The number of DSpace sessions isn't even that high: + +``` +$ cat /home/cgspace.cgiar.org/log/dspace.log.2017-12-01 | grep -o -E 'session_id=[A-Z0-9]{32}' | sort -n | uniq | wc -l +5815 +``` + +- Connections in the last two hours: + +``` +# cat /var/log/nginx/rest.log /var/log/nginx/rest.log.1 /var/log/nginx/access.log /var/log/nginx/access.log.1 /var/log/nginx/library-access.log /var/log/nginx/library-access.log.1 | grep -E "1/Dec/2017:(09|10)" | awk '{print $1}' | sort -n | uniq -c | sort -h | tail + 78 93.160.60.22 + 101 40.77.167.122 + 113 66.249.66.70 + 129 157.55.39.206 + 130 157.55.39.235 + 135 40.77.167.58 + 164 68.180.229.254 + 177 87.100.118.220 + 188 66.249.66.90 + 314 2.86.122.76 +``` + +- What the fuck is going on? +- I've never seen this 2.86.122.76 before, it has made quite a few unique Tomcat sessions today: + +``` +$ grep 2.86.122.76 /home/cgspace.cgiar.org/log/dspace.log.2017-12-01 | grep -o -E 'session_id=[A-Z0-9]{32}' | sort -n | uniq | wc -l +822 +``` + +- Appears to be some new bot: + +``` +2.86.122.76 - - [01/Dec/2017:09:02:53 +0000] "GET /handle/10568/78444?show=full HTTP/1.1" 200 29307 "-" "Mozilla/3.0 (compatible; Indy Library)" +``` + +- I restarted Tomcat and everything came back up +- I can add Indy Library to the Tomcat crawler session manager valve but it would be nice if I could simply remap the useragent in nginx +- I will also add 'Drupal' to the Tomcat crawler session manager valve because there are Drupals out there harvesting and they should be considered as bots + +``` +# cat /var/log/nginx/rest.log /var/log/nginx/rest.log.1 /var/log/nginx/access.log /var/log/nginx/access.log.1 /var/log/nginx/library-access.log /var/log/nginx/library-access.log.1 | grep -E "1/Dec/2017" | grep Drupal | awk '{print $1}' | sort -n | uniq -c | sort -h | tail + 3 54.75.205.145 + 6 70.32.83.92 + 14 2a01:7e00::f03c:91ff:fe18:7396 + 46 2001:4b99:1:1:216:3eff:fe2c:dc6c + 319 2001:4b99:1:1:216:3eff:fe76:205b +``` diff --git a/public/2015-11/index.html b/public/2015-11/index.html index faf6046b7..77f8e18a4 100644 --- a/public/2015-11/index.html +++ b/public/2015-11/index.html @@ -26,7 +26,7 @@ $ psql -c 'SELECT * from pg_stat_activity;' | grep idle | grep -c cgspac - + @@ -64,7 +64,7 @@ $ psql -c 'SELECT * from pg_stat_activity;' | grep idle | grep -c cgspac "url": "https://alanorth.github.io/cgspace-notes/2015-11/", "wordCount": "798", "datePublished": "2015-11-23T17:00:57+03:00", - "dateModified": "2016-09-28T17:02:30+03:00", + "dateModified": "2015-11-23T17:00:57+03:00", "author": { "@type": "Person", "name": "Alan Orth" @@ -293,6 +293,8 @@ db.statementpool = true
    +
  1. December, 2017
    2. +
  2. November, 2017
  3. October, 2017
    4. @@ -301,8 +303,6 @@ db.statementpool = true
  4. September, 2017
    5. -
  5. August, 2017
    6. -
diff --git a/public/2015-12/index.html b/public/2015-12/index.html index 46e33a861..a567dad5f 100644 --- a/public/2015-12/index.html +++ b/public/2015-12/index.html @@ -27,7 +27,7 @@ Replace lzop with xz in log compression cron jobs on DSpace Test—it uses less - + @@ -66,7 +66,7 @@ Replace lzop with xz in log compression cron jobs on DSpace Test—it uses less "url": "https://alanorth.github.io/cgspace-notes/2015-12/", "wordCount": "753", "datePublished": "2015-12-02T13:18:00+03:00", - "dateModified": "2017-01-09T16:18:07+02:00", + "dateModified": "2015-12-02T13:18:00+03:00", "author": { "@type": "Person", "name": "Alan Orth" @@ -310,6 +310,8 @@ $ curl -o /dev/null -s -w %{time_total}\\n https://cgspace.cgiar.org/rest/handle
    +
  1. December, 2017
    2. +
  2. November, 2017
  3. October, 2017
    4. @@ -318,8 +320,6 @@ $ curl -o /dev/null -s -w %{time_total}\\n https://cgspace.cgiar.org/rest/handle
  4. September, 2017
    5. -
  5. August, 2017
    6. -
diff --git a/public/2016-01/index.html b/public/2016-01/index.html index 1c0764a1b..2c33b6f78 100644 --- a/public/2016-01/index.html +++ b/public/2016-01/index.html @@ -22,7 +22,7 @@ Update GitHub wiki for documentation of maintenance tasks. - + @@ -56,7 +56,7 @@ Update GitHub wiki for documentation of maintenance tasks. "url": "https://alanorth.github.io/cgspace-notes/2016-01/", "wordCount": "466", "datePublished": "2016-01-13T13:18:00+03:00", - "dateModified": "2017-01-09T16:18:07+02:00", + "dateModified": "2016-01-13T13:18:00+03:00", "author": { "@type": "Person", "name": "Alan Orth" @@ -225,6 +225,8 @@ $ find SimpleArchiveForBio/ -iname “*.pdf” -exec basename {} \; | so
    +
  1. December, 2017
    2. +
  2. November, 2017
  3. October, 2017
    4. @@ -233,8 +235,6 @@ $ find SimpleArchiveForBio/ -iname “*.pdf” -exec basename {} \; | so
  4. September, 2017
    5. -
  5. August, 2017
    6. -
diff --git a/public/2016-02/index.html b/public/2016-02/index.html index 78eaab347..60d77fd59 100644 --- a/public/2016-02/index.html +++ b/public/2016-02/index.html @@ -29,7 +29,7 @@ Also, lots of things like “COTE D`LVOIRE” and “COTE D IVOIRE&r - + @@ -70,7 +70,7 @@ Also, lots of things like “COTE D`LVOIRE” and “COTE D IVOIRE&r "url": "https://alanorth.github.io/cgspace-notes/2016-02/", "wordCount": "1657", "datePublished": "2016-02-05T13:18:00+03:00", - "dateModified": "2017-01-09T16:18:07+02:00", + "dateModified": "2016-02-05T13:18:00+03:00", "author": { "@type": "Person", "name": "Alan Orth" @@ -478,6 +478,8 @@ Bitstream: tést señora alimentación.pdf
    +
  1. December, 2017
    2. +
  2. November, 2017
  3. October, 2017
    4. @@ -486,8 +488,6 @@ Bitstream: tést señora alimentación.pdf
  4. September, 2017
    5. -
  5. August, 2017
    6. -
diff --git a/public/2016-03/index.html b/public/2016-03/index.html index 4268fc329..5ec5acc98 100644 --- a/public/2016-03/index.html +++ b/public/2016-03/index.html @@ -22,7 +22,7 @@ Reinstall my local (Mac OS X) DSpace stack with Tomcat 7, PostgreSQL 9.3, and Ja - + @@ -56,7 +56,7 @@ Reinstall my local (Mac OS X) DSpace stack with Tomcat 7, PostgreSQL 9.3, and Ja "url": "https://alanorth.github.io/cgspace-notes/2016-03/", "wordCount": "1581", "datePublished": "2016-03-02T16:50:00+03:00", - "dateModified": "2017-01-09T16:18:07+02:00", + "dateModified": "2016-03-02T16:50:00+03:00", "author": { "@type": "Person", "name": "Alan Orth" @@ -378,6 +378,8 @@ Reinstall my local (Mac OS X) DSpace stack with Tomcat 7, PostgreSQL 9.3, and Ja
    +
  1. December, 2017
    2. +
  2. November, 2017
  3. October, 2017
    4. @@ -386,8 +388,6 @@ Reinstall my local (Mac OS X) DSpace stack with Tomcat 7, PostgreSQL 9.3, and Ja
  4. September, 2017
    5. -
  5. August, 2017
    6. -
diff --git a/public/2016-04/index.html b/public/2016-04/index.html index d4f0996be..84062c4a5 100644 --- a/public/2016-04/index.html +++ b/public/2016-04/index.html @@ -24,7 +24,7 @@ Also, I noticed the checker log has some errors we should pay attention to: - + @@ -60,7 +60,7 @@ Also, I noticed the checker log has some errors we should pay attention to: "url": "https://alanorth.github.io/cgspace-notes/2016-04/", "wordCount": "2006", "datePublished": "2016-04-04T11:06:00+03:00", - "dateModified": "2016-09-28T17:02:30+03:00", + "dateModified": "2016-04-04T11:06:00+03:00", "author": { "@type": "Person", "name": "Alan Orth" @@ -595,6 +595,8 @@ dspace.log.2016-04-27:7271
    +
  1. December, 2017
    2. +
  2. November, 2017
  3. October, 2017
    4. @@ -603,8 +605,6 @@ dspace.log.2016-04-27:7271
  4. September, 2017
    5. -
  5. August, 2017
    6. -
diff --git a/public/2016-05/index.html b/public/2016-05/index.html index b2c3f01ac..fbea54dd6 100644 --- a/public/2016-05/index.html +++ b/public/2016-05/index.html @@ -26,7 +26,7 @@ There are 3,000 IPs accessing the REST API in a 24-hour period! - + @@ -64,7 +64,7 @@ There are 3,000 IPs accessing the REST API in a 24-hour period! "url": "https://alanorth.github.io/cgspace-notes/2016-05/", "wordCount": "1349", "datePublished": "2016-05-01T23:06:00+03:00", - "dateModified": "2017-01-09T16:18:07+02:00", + "dateModified": "2016-05-01T23:06:00+03:00", "author": { "@type": "Person", "name": "Alan Orth" @@ -445,6 +445,8 @@ sys 0m20.540s
    +
  1. December, 2017
    2. +
  2. November, 2017
  3. October, 2017
    4. @@ -453,8 +455,6 @@ sys 0m20.540s
  4. September, 2017
    5. -
  5. August, 2017
    6. -
diff --git a/public/2016-06/index.html b/public/2016-06/index.html index f60ca9423..8b56224de 100644 --- a/public/2016-06/index.html +++ b/public/2016-06/index.html @@ -25,7 +25,7 @@ Working on second phase of metadata migration, looks like this will work for mov - + @@ -62,7 +62,7 @@ Working on second phase of metadata migration, looks like this will work for mov "url": "https://alanorth.github.io/cgspace-notes/2016-06/", "wordCount": "1549", "datePublished": "2016-06-01T10:53:00+03:00", - "dateModified": "2017-01-09T16:18:07+02:00", + "dateModified": "2016-06-01T10:53:00+03:00", "author": { "@type": "Person", "name": "Alan Orth" @@ -490,6 +490,8 @@ $ ./delete-metadata-values.py -f dc.contributor.corporate -i Corporate-Authors-D
    +
  1. December, 2017
    2. +
  2. November, 2017
  3. October, 2017
    4. @@ -498,8 +500,6 @@ $ ./delete-metadata-values.py -f dc.contributor.corporate -i Corporate-Authors-D
  4. September, 2017
    5. -
  5. August, 2017
    6. -
diff --git a/public/2016-07/index.html b/public/2016-07/index.html index 96e583e55..7f6d02dfa 100644 --- a/public/2016-07/index.html +++ b/public/2016-07/index.html @@ -33,7 +33,7 @@ In this case the select query was showing 95 results before the update - + @@ -78,7 +78,7 @@ In this case the select query was showing 95 results before the update "url": "https://alanorth.github.io/cgspace-notes/2016-07/", "wordCount": "866", "datePublished": "2016-07-01T10:53:00+03:00", - "dateModified": "2017-01-09T16:18:07+02:00", + "dateModified": "2016-07-01T10:53:00+03:00", "author": { "@type": "Person", "name": "Alan Orth" @@ -408,6 +408,8 @@ discovery.index.authority.ignore-variants=true
    +
  1. December, 2017
    2. +
  2. November, 2017
  3. October, 2017
    4. @@ -416,8 +418,6 @@ discovery.index.authority.ignore-variants=true
  4. September, 2017
    5. -
  5. August, 2017
    6. -
diff --git a/public/2016-08/index.html b/public/2016-08/index.html index 0ab697471..5f430b157 100644 --- a/public/2016-08/index.html +++ b/public/2016-08/index.html @@ -30,7 +30,7 @@ $ git rebase -i dspace-5.5 - + @@ -72,7 +72,7 @@ $ git rebase -i dspace-5.5 "url": "https://alanorth.github.io/cgspace-notes/2016-08/", "wordCount": "1514", "datePublished": "2016-08-01T15:53:00+03:00", - "dateModified": "2017-01-09T16:18:07+02:00", + "dateModified": "2016-08-01T15:53:00+03:00", "author": { "@type": "Person", "name": "Alan Orth" @@ -481,6 +481,8 @@ $ JAVA_OPTS="-Dfile.encoding=UTF-8 -Xmx512m" /home/cgspace.cgiar.org/b
    +
  1. December, 2017
    2. +
  2. November, 2017
  3. October, 2017
    4. @@ -489,8 +491,6 @@ $ JAVA_OPTS="-Dfile.encoding=UTF-8 -Xmx512m" /home/cgspace.cgiar.org/b
  4. September, 2017
    5. -
  5. August, 2017
    6. -
diff --git a/public/2016-09/index.html b/public/2016-09/index.html index 58e845523..053f8b096 100644 --- a/public/2016-09/index.html +++ b/public/2016-09/index.html @@ -26,7 +26,7 @@ $ ldapsearch -x -H ldaps://svcgroot2.cgiarad.org:3269/ -b "dc=cgiarad,dc=or - + @@ -64,7 +64,7 @@ $ ldapsearch -x -H ldaps://svcgroot2.cgiarad.org:3269/ -b "dc=cgiarad,dc=or "url": "https://alanorth.github.io/cgspace-notes/2016-09/", "wordCount": "3298", "datePublished": "2016-09-01T15:53:00+03:00", - "dateModified": "2017-01-09T16:18:07+02:00", + "dateModified": "2016-09-01T15:53:00+03:00", "author": { "@type": "Person", "name": "Alan Orth" @@ -781,6 +781,8 @@ $ ./delete-metadata-values.py -i ilrisubjects-delete-13.csv -f cg.subject.ilri -
    +
  1. December, 2017
    2. +
  2. November, 2017
  3. October, 2017
    4. @@ -789,8 +791,6 @@ $ ./delete-metadata-values.py -i ilrisubjects-delete-13.csv -f cg.subject.ilri -
  4. September, 2017
    5. -
  5. August, 2017
    6. -
diff --git a/public/2016-10/index.html b/public/2016-10/index.html index 8f79b18c7..e02acc5aa 100644 --- a/public/2016-10/index.html +++ b/public/2016-10/index.html @@ -30,7 +30,7 @@ I exported a random item’s metadata as CSV, deleted all columns except id - + @@ -72,7 +72,7 @@ I exported a random item’s metadata as CSV, deleted all columns except id "url": "https://alanorth.github.io/cgspace-notes/2016-10/", "wordCount": "1828", "datePublished": "2016-10-03T15:53:00+03:00", - "dateModified": "2017-01-10T16:21:47+02:00", + "dateModified": "2016-10-03T15:53:00+03:00", "author": { "@type": "Person", "name": "Alan Orth" @@ -467,6 +467,8 @@ dspace=# update metadatavalue set text_value = regexp_replace(text_value, 'http:
    +
  1. December, 2017
    2. +
  2. November, 2017
  3. October, 2017
    4. @@ -475,8 +477,6 @@ dspace=# update metadatavalue set text_value = regexp_replace(text_value, 'http:
  4. September, 2017
    5. -
  5. August, 2017
    6. -
diff --git a/public/2016-11/index.html b/public/2016-11/index.html index 5a45fd747..aa801584f 100644 --- a/public/2016-11/index.html +++ b/public/2016-11/index.html @@ -22,7 +22,7 @@ Add dc.type to the output options for Atmire’s Listings and Reports module - + @@ -56,7 +56,7 @@ Add dc.type to the output options for Atmire’s Listings and Reports module "url": "https://alanorth.github.io/cgspace-notes/2016-11/", "wordCount": "2825", "datePublished": "2016-11-01T09:21:00+03:00", - "dateModified": "2017-01-10T16:21:47+02:00", + "dateModified": "2016-11-01T09:21:00+03:00", "author": { "@type": "Person", "name": "Alan Orth" @@ -683,6 +683,8 @@ org.dspace.discovery.SearchServiceException: Error executing query
    +
  1. December, 2017
    2. +
  2. November, 2017
  3. October, 2017
    4. @@ -691,8 +693,6 @@ org.dspace.discovery.SearchServiceException: Error executing query
  4. September, 2017
    5. -
  5. August, 2017
    6. -
diff --git a/public/2016-12/index.html b/public/2016-12/index.html index 2f8ae0892..aecb9d7ff 100644 --- a/public/2016-12/index.html +++ b/public/2016-12/index.html @@ -34,7 +34,7 @@ Another worrying error from dspace.log is: - + @@ -80,7 +80,7 @@ Another worrying error from dspace.log is: "url": "https://alanorth.github.io/cgspace-notes/2016-12/", "wordCount": "4078", "datePublished": "2016-12-02T10:43:00+03:00", - "dateModified": "2017-09-19T16:07:20+03:00", + "dateModified": "2016-12-02T10:43:00+03:00", "author": { "@type": "Person", "name": "Alan Orth" @@ -919,6 +919,8 @@ $ exit
    +
  1. December, 2017
    2. +
  2. November, 2017
  3. October, 2017
    4. @@ -927,8 +929,6 @@ $ exit
  4. September, 2017
    5. -
  5. August, 2017
    6. -
diff --git a/public/2017-01/index.html b/public/2017-01/index.html index 4fbc0c1cc..1945680e5 100644 --- a/public/2017-01/index.html +++ b/public/2017-01/index.html @@ -22,7 +22,7 @@ I asked on the dspace-tech mailing list because it seems to be broken, and actua - + @@ -56,7 +56,7 @@ I asked on the dspace-tech mailing list because it seems to be broken, and actua "url": "https://alanorth.github.io/cgspace-notes/2017-01/", "wordCount": "1594", "datePublished": "2017-01-02T10:43:00+03:00", - "dateModified": "2017-01-29T13:18:32+02:00", + "dateModified": "2017-01-02T10:43:00+03:00", "author": { "@type": "Person", "name": "Alan Orth" @@ -464,6 +464,8 @@ $ gs -sDEVICE=pdfwrite -dCompatibilityLevel=1.4 -dPDFSETTINGS=/ebook -dNOPAUSE -
    +
  1. December, 2017
    2. +
  2. November, 2017
  3. October, 2017
    4. @@ -472,8 +474,6 @@ $ gs -sDEVICE=pdfwrite -dCompatibilityLevel=1.4 -dPDFSETTINGS=/ebook -dNOPAUSE -
  4. September, 2017
    5. -
  5. August, 2017
    6. -
diff --git a/public/2017-02/index.html b/public/2017-02/index.html index 92fbc2643..d85e3a858 100644 --- a/public/2017-02/index.html +++ b/public/2017-02/index.html @@ -36,7 +36,7 @@ Looks like we’ll be using cg.identifier.ccafsprojectpii as the field name - + @@ -84,7 +84,7 @@ Looks like we’ll be using cg.identifier.ccafsprojectpii as the field name "url": "https://alanorth.github.io/cgspace-notes/2017-02/", "wordCount": "2028", "datePublished": "2017-02-07T07:04:52-08:00", - "dateModified": "2017-02-28T22:58:29+02:00", + "dateModified": "2017-02-07T07:04:52-08:00", "author": { "@type": "Person", "name": "Alan Orth" @@ -536,6 +536,8 @@ COPY 1968
    +
  1. December, 2017
    2. +
  2. November, 2017
  3. October, 2017
    4. @@ -544,8 +546,6 @@ COPY 1968
  4. September, 2017
    5. -
  5. August, 2017
    6. -
diff --git a/public/2017-03/index.html b/public/2017-03/index.html index 0f56a95a7..8ca7e8af3 100644 --- a/public/2017-03/index.html +++ b/public/2017-03/index.html @@ -38,7 +38,7 @@ $ identify ~/Desktop/alc_contrastes_desafios.jpg - + @@ -88,7 +88,7 @@ $ identify ~/Desktop/alc_contrastes_desafios.jpg "url": "https://alanorth.github.io/cgspace-notes/2017-03/", "wordCount": "1538", "datePublished": "2017-03-01T17:08:52+02:00", - "dateModified": "2017-03-31T05:36:10+03:00", + "dateModified": "2017-03-01T17:08:52+02:00", "author": { "@type": "Person", "name": "Alan Orth" @@ -450,6 +450,8 @@ $ ./delete-metadata-values.py -i Investors-Delete-121.csv -f dc.description.spon
    +
  1. December, 2017
    2. +
  2. November, 2017
  3. October, 2017
    4. @@ -458,8 +460,6 @@ $ ./delete-metadata-values.py -i Investors-Delete-121.csv -f dc.description.spon
  4. September, 2017
    5. -
  5. August, 2017
    6. -
diff --git a/public/2017-04/index.html b/public/2017-04/index.html index 51a89e382..1836b709f 100644 --- a/public/2017-04/index.html +++ b/public/2017-04/index.html @@ -31,7 +31,7 @@ $ [dspace]/bin/dspace filter-media -f -i 10568/16498 -p "ImageMagick PDF Th - + @@ -74,7 +74,7 @@ $ [dspace]/bin/dspace filter-media -f -i 10568/16498 -p "ImageMagick PDF Th "url": "https://alanorth.github.io/cgspace-notes/2017-04/", "wordCount": "2917", "datePublished": "2017-04-02T17:08:52+02:00", - "dateModified": "2017-04-26T13:35:10+03:00", + "dateModified": "2017-04-02T17:08:52+02:00", "author": { "@type": "Person", "name": "Alan Orth" @@ -729,6 +729,8 @@ $ gem install compass -v 1.0.3
    +
  1. December, 2017
    2. +
  2. November, 2017
  3. October, 2017
    4. @@ -737,8 +739,6 @@ $ gem install compass -v 1.0.3
  4. September, 2017
    5. -
  5. August, 2017
    6. -
diff --git a/public/2017-05/index.html b/public/2017-05/index.html index ecdc3177c..3fc2fa0ff 100644 --- a/public/2017-05/index.html +++ b/public/2017-05/index.html @@ -14,7 +14,7 @@ - + @@ -40,7 +40,7 @@ "url": "https://alanorth.github.io/cgspace-notes/2017-05/", "wordCount": "2398", "datePublished": "2017-05-01T16:21:52+02:00", - "dateModified": "2017-09-10T17:46:54+03:00", + "dateModified": "2017-05-01T16:21:52+02:00", "author": { "@type": "Person", "name": "Alan Orth" @@ -483,6 +483,8 @@ UPDATE 187
    +
  1. December, 2017
    2. +
  2. November, 2017
  3. October, 2017
    4. @@ -491,8 +493,6 @@ UPDATE 187
  4. September, 2017
    5. -
  5. August, 2017
    6. -
diff --git a/public/2017-06/index.html b/public/2017-06/index.html index 92d1a02e3..b4be14dd1 100644 --- a/public/2017-06/index.html +++ b/public/2017-06/index.html @@ -14,7 +14,7 @@ - + @@ -40,7 +40,7 @@ "url": "https://alanorth.github.io/cgspace-notes/2017-06/", "wordCount": "1261", "datePublished": "2017-06-01T10:14:52+03:00", - "dateModified": "2017-06-30T18:34:51+03:00", + "dateModified": "2017-06-01T10:14:52+03:00", "author": { "@type": "Person", "name": "Alan Orth" @@ -301,6 +301,8 @@ $ JAVA_OPTS="-Xmx1024m -Dfile.encoding=UTF-8" [dspace]/bin/dspace impo
    +
  1. December, 2017
    2. +
  2. November, 2017
  3. October, 2017
    4. @@ -309,8 +311,6 @@ $ JAVA_OPTS="-Xmx1024m -Dfile.encoding=UTF-8" [dspace]/bin/dspace impo
  4. September, 2017
    5. -
  5. August, 2017
    6. -
diff --git a/public/2017-07/index.html b/public/2017-07/index.html index 79060644e..90747fd06 100644 --- a/public/2017-07/index.html +++ b/public/2017-07/index.html @@ -28,7 +28,7 @@ We can use PostgreSQL’s extended output format (-x) plus sed to format the - + @@ -68,7 +68,7 @@ We can use PostgreSQL’s extended output format (-x) plus sed to format the "url": "https://alanorth.github.io/cgspace-notes/2017-07/", "wordCount": "1151", "datePublished": "2017-07-01T18:03:52+03:00", - "dateModified": "2017-08-01T08:55:37+03:00", + "dateModified": "2017-07-01T18:03:52+03:00", "author": { "@type": "Person", "name": "Alan Orth" @@ -336,6 +336,8 @@ delete from metadatavalue where resource_type_id=2 and metadata_field_id=235 and
    +
  1. December, 2017
    2. +
  2. November, 2017
  3. October, 2017
    4. @@ -344,8 +346,6 @@ delete from metadatavalue where resource_type_id=2 and metadata_field_id=235 and
  4. September, 2017
    5. -
  5. August, 2017
    6. -
diff --git a/public/2017-08/index.html b/public/2017-08/index.html index 18c180c58..3327f3321 100644 --- a/public/2017-08/index.html +++ b/public/2017-08/index.html @@ -38,7 +38,7 @@ Then I cleaned up the author authorities and HTML characters in OpenRefine and s - + @@ -88,7 +88,7 @@ Then I cleaned up the author authorities and HTML characters in OpenRefine and s "url": "https://alanorth.github.io/cgspace-notes/2017-08/", "wordCount": "3542", "datePublished": "2017-08-01T11:51:52+03:00", - "dateModified": "2017-09-10T19:18:52+03:00", + "dateModified": "2017-08-01T11:51:52+03:00", "author": { "@type": "Person", "name": "Alan Orth" @@ -622,6 +622,8 @@ org.apache.commons.dbcp.SQLNestedException: Cannot get a connection, pool error
    +
  1. December, 2017
    2. +
  2. November, 2017
  3. October, 2017
    4. @@ -630,8 +632,6 @@ org.apache.commons.dbcp.SQLNestedException: Cannot get a connection, pool error
  4. September, 2017
    5. -
  5. August, 2017
    6. -
diff --git a/public/2017-09/index.html b/public/2017-09/index.html index fe1dd80fd..1b39239a6 100644 --- a/public/2017-09/index.html +++ b/public/2017-09/index.html @@ -26,7 +26,7 @@ Ask Sisay to clean up the WLE approvers a bit, as Marianne’s user account - + @@ -64,7 +64,7 @@ Ask Sisay to clean up the WLE approvers a bit, as Marianne’s user account "url": "https://alanorth.github.io/cgspace-notes/2017-09/", "wordCount": "4199", "datePublished": "2017-09-07T16:54:52+07:00", - "dateModified": "2017-09-28T07:56:11+03:00", + "dateModified": "2017-09-07T16:54:52+07:00", "author": { "@type": "Person", "name": "Alan Orth" @@ -794,6 +794,8 @@ Cert Status: good
    +
  1. December, 2017
    2. +
  2. November, 2017
  3. October, 2017
    4. @@ -802,8 +804,6 @@ Cert Status: good
  4. September, 2017
    5. -
  5. August, 2017
    6. -
diff --git a/public/2017-10/index.html b/public/2017-10/index.html index c626b5d31..1547a2cd6 100644 --- a/public/2017-10/index.html +++ b/public/2017-10/index.html @@ -28,7 +28,7 @@ Add Katherine Lutz to the groups for content submission and edit steps of the CG - + @@ -68,7 +68,7 @@ Add Katherine Lutz to the groups for content submission and edit steps of the CG "url": "https://alanorth.github.io/cgspace-notes/2017-10/", "wordCount": "2613", "datePublished": "2017-10-01T08:07:54+03:00", - "dateModified": "2017-11-02T16:13:10+02:00", + "dateModified": "2017-10-01T08:07:54+03:00", "author": { "@type": "Person", "name": "Alan Orth" @@ -559,6 +559,8 @@ session_id=6C30F10B4351A4ED83EC6ED50AFD6B6A
    +
  1. December, 2017
    2. +
  2. November, 2017
  3. October, 2017
    4. @@ -567,8 +569,6 @@ session_id=6C30F10B4351A4ED83EC6ED50AFD6B6A
  4. September, 2017
    5. -
  5. August, 2017
    6. -
diff --git a/public/2017-11/index.html b/public/2017-11/index.html index 009260408..6955f6816 100644 --- a/public/2017-11/index.html +++ b/public/2017-11/index.html @@ -38,7 +38,7 @@ COPY 54701 - + @@ -88,7 +88,7 @@ COPY 54701 "url": "https://alanorth.github.io/cgspace-notes/2017-11/", "wordCount": "5428", "datePublished": "2017-11-02T09:37:54+02:00", - "dateModified": "2017-11-29T17:26:17+03:00", + "dateModified": "2017-11-02T09:37:54+02:00", "author": { "@type": "Person", "name": "Alan Orth" @@ -1203,6 +1203,8 @@ $ cat dspace.log.2017-11-28 | grep -o -E 'session_id=[A-Z0-9]{32}' | sort -n | u
    +
  1. December, 2017
    2. +
  2. November, 2017
  3. October, 2017
    4. @@ -1211,8 +1213,6 @@ $ cat dspace.log.2017-11-28 | grep -o -E 'session_id=[A-Z0-9]{32}' | sort -n | u
  4. September, 2017
    5. -
  5. August, 2017
    6. -
diff --git a/public/2017-12/index.html b/public/2017-12/index.html new file mode 100644 index 000000000..f98bf5dad --- /dev/null +++ b/public/2017-12/index.html @@ -0,0 +1,270 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + December, 2017 | CGSpace Notes + + + + + + + + + + + + + + + + +
+
+ +
+
+ +
+
+

CGSpace Notes

+

Documenting day-to-day work on the CGSpace repository.

+
+
+ +
+
+
+ + + + +
+
+

December, 2017

+

by Alan Orth in + +  + +

+
+

2017-12-01

+ +
    +
  • Uptime Robot noticed that CGSpace went down
    • +
  • The logs say “Timeout waiting for idle object”
    • +
  • PostgreSQL activity says there are 115 connections currently
    • +
  • The list of connections to XMLUI and REST API for today:
    • +
+ +

+ +
# cat /var/log/nginx/rest.log  /var/log/nginx/rest.log.1  /var/log/nginx/access.log /var/log/nginx/access.log.1 /var/log/nginx/library-access.log /var/log/nginx/library-access.log.1 | grep -E "1/Dec/2017" | awk '{print $1}' | sort -n | uniq -c | sort -h | tail
+    763 2.86.122.76
+    907 207.46.13.94
+   1018 157.55.39.206
+   1021 157.55.39.235
+   1407 66.249.66.70
+   1411 104.196.152.243
+   1503 50.116.102.77
+   1805 66.249.66.90
+   4007 70.32.83.92
+   6061 45.5.184.196
+
+ +
    +
  • The number of DSpace sessions isn’t even that high:
    • +
+ +
$ cat /home/cgspace.cgiar.org/log/dspace.log.2017-12-01 | grep -o -E 'session_id=[A-Z0-9]{32}' | sort -n | uniq | wc -l
+5815
+
+ +
    +
  • Connections in the last two hours:
    • +
+ +
# cat /var/log/nginx/rest.log  /var/log/nginx/rest.log.1  /var/log/nginx/access.log /var/log/nginx/access.log.1 /var/log/nginx/library-access.log /var/log/nginx/library-access.log.1 | grep -E "1/Dec/2017:(09|10)" | awk '{print $1}' | sort -n | uniq -c | sort -h | tail                                                      
+     78 93.160.60.22
+    101 40.77.167.122
+    113 66.249.66.70
+    129 157.55.39.206
+    130 157.55.39.235
+    135 40.77.167.58
+    164 68.180.229.254
+    177 87.100.118.220
+    188 66.249.66.90
+    314 2.86.122.76
+
+ +
    +
  • What the fuck is going on?
    • +
  • I’ve never seen this 2.86.122.76 before, it has made quite a few unique Tomcat sessions today:
    • +
+ +
$ grep 2.86.122.76 /home/cgspace.cgiar.org/log/dspace.log.2017-12-01 | grep -o -E 'session_id=[A-Z0-9]{32}' | sort -n | uniq | wc -l
+822
+
+ +
    +
  • Appears to be some new bot:
    • +
+ +
2.86.122.76 - - [01/Dec/2017:09:02:53 +0000] "GET /handle/10568/78444?show=full HTTP/1.1" 200 29307 "-" "Mozilla/3.0 (compatible; Indy Library)"
+
+ +
    +
  • I restarted Tomcat and everything came back up
    • +
  • I can add Indy Library to the Tomcat crawler session manager valve but it would be nice if I could simply remap the useragent in nginx
    • +
  • I will also add ‘Drupal’ to the Tomcat crawler session manager valve because there are Drupals out there harvesting and they should be considered as bots
    • +
+ +
# cat /var/log/nginx/rest.log  /var/log/nginx/rest.log.1  /var/log/nginx/access.log /var/log/nginx/access.log.1 /var/log/nginx/library-access.log /var/log/nginx/library-access.log.1 | grep -E "1/Dec/2017" | grep Drupal | awk '{print $1}' | sort -n | uniq -c | sort -h | tail
+      3 54.75.205.145
+      6 70.32.83.92
+     14 2a01:7e00::f03c:91ff:fe18:7396
+     46 2001:4b99:1:1:216:3eff:fe2c:dc6c
+    319 2001:4b99:1:1:216:3eff:fe76:205b
+
+ + + + + +
+ + + +
+ + + + +
+
+ + + + + + diff --git a/public/categories/notes/index.html b/public/categories/notes/index.html index 02bb94eda..c8f6b7f45 100644 --- a/public/categories/notes/index.html +++ b/public/categories/notes/index.html @@ -130,6 +130,8 @@
    +
  1. December, 2017
    2. +
  2. November, 2017
  3. October, 2017
    4. @@ -138,8 +140,6 @@
  4. September, 2017
    5. -
  5. August, 2017
    6. -
diff --git a/public/cgiar-library-migration/index.html b/public/cgiar-library-migration/index.html index 6962e6574..2fa176e17 100644 --- a/public/cgiar-library-migration/index.html +++ b/public/cgiar-library-migration/index.html @@ -14,7 +14,7 @@ - + @@ -40,7 +40,7 @@ "url": "https://alanorth.github.io/cgspace-notes/cgiar-library-migration/", "wordCount": "1278", "datePublished": "2017-09-18T16:38:35+03:00", - "dateModified": "2017-09-28T12:00:49+03:00", + "dateModified": "2017-09-18T16:38:35+03:00", "author": { "@type": "Person", "name": "Alan Orth" @@ -336,6 +336,8 @@ dspace=# select setval('handle_seq',86873);
    +
  1. December, 2017
    2. +
  2. November, 2017
  3. October, 2017
    4. @@ -344,8 +346,6 @@ dspace=# select setval('handle_seq',86873);
  4. September, 2017
    5. -
  5. August, 2017
    6. -
diff --git a/public/index.html b/public/index.html index ae470aa3c..622f57a4c 100644 --- a/public/index.html +++ b/public/index.html @@ -12,7 +12,7 @@ - + @@ -39,7 +39,7 @@ "@type": "Person", "name": "Alan Orth" }, - "dateModified": "2017-11-02T09:37:54+02:00", + "dateModified": "2017-12-01T13:53:54+03:00", "keywords": "notes,notes,", "description": "Documenting day-to-day work on the [CGSpace](https://cgspace.cgiar.org) repository." } @@ -97,6 +97,33 @@ +
+
+

December, 2017

+

by Alan Orth in + +  + +

+
+

2017-12-01

+ + + +

+ Read more → +
+ + + + + +

November, 2017

@@ -365,48 +392,6 @@ COPY 54701 - -
-
-

March, 2017

-

by Alan Orth in - -  - -

-
-

2017-03-01

- -
    -
  • Run the 279 CIAT author corrections on CGSpace
    • -
- -

2017-03-02

- -
    -
  • Skype with Michael and Peter, discussing moving the CGIAR Library to CGSpace
    • -
  • CGIAR people possibly open to moving content, redirecting library.cgiar.org to CGSpace and letting CGSpace resolve their handles
    • -
  • They might come in at the top level in one “CGIAR System” community, or with several communities
    • -
  • I need to spend a bit of time looking at the multiple handle support in DSpace and see if new content can be minted in both handles, or just one?
    • -
  • Need to send Peter and Michael some notes about this in a few days
    • -
  • Also, need to consider talking to Atmire about hiring them to bring ORCiD metadata to REST / OAI
    • -
  • Filed an issue on DSpace issue tracker for the filter-media bug that causes it to process JPGs even when limiting to the PDF thumbnail plugin: DS-3516
    • -
  • Discovered that the ImageMagic filter-media plugin creates JPG thumbnails with the CMYK colorspace when the source PDF is using CMYK
    • -
  • Interestingly, it seems DSpace 4.x’s thumbnails were sRGB, but forcing regeneration using DSpace 5.x’s ImageMagick plugin creates CMYK JPGs if the source PDF was CMYK (see 10568⁄51999):
    • -
- -
$ identify ~/Desktop/alc_contrastes_desafios.jpg
-/Users/aorth/Desktop/alc_contrastes_desafios.jpg JPEG 464x600 464x600+0+0 8-bit CMYK 168KB 0.000u 0:00.000
-
- -

- Read more → -
- - - - -