From d1278a67d80de5b2af30828e73cf18533bddd163 Mon Sep 17 00:00:00 2001 From: Alan Orth Date: Wed, 4 Jan 2023 17:08:14 +0300 Subject: [PATCH] Add notes for 2023-01-04 --- content/posts/2023-01.md | 27 +++++++++++++ docs/2022-11/index.html | 8 ++-- docs/2022-12/index.html | 4 +- docs/2023-01/index.html | 51 +++++++++++++++++++++++-- docs/categories/index.html | 2 +- docs/categories/notes/index.html | 2 +- docs/categories/notes/page/2/index.html | 2 +- docs/categories/notes/page/3/index.html | 2 +- docs/categories/notes/page/4/index.html | 2 +- docs/categories/notes/page/5/index.html | 2 +- docs/categories/notes/page/6/index.html | 2 +- docs/categories/notes/page/7/index.html | 2 +- docs/index.html | 2 +- docs/page/2/index.html | 2 +- docs/page/3/index.html | 2 +- docs/page/4/index.html | 2 +- docs/page/5/index.html | 2 +- docs/page/6/index.html | 2 +- docs/page/7/index.html | 2 +- docs/page/8/index.html | 2 +- docs/page/9/index.html | 2 +- docs/posts/index.html | 2 +- docs/posts/page/2/index.html | 2 +- docs/posts/page/3/index.html | 2 +- docs/posts/page/4/index.html | 2 +- docs/posts/page/5/index.html | 2 +- docs/posts/page/6/index.html | 2 +- docs/posts/page/7/index.html | 2 +- docs/posts/page/8/index.html | 2 +- docs/posts/page/9/index.html | 2 +- docs/sitemap.xml | 14 +++---- 31 files changed, 114 insertions(+), 42 deletions(-) diff --git a/content/posts/2023-01.md b/content/posts/2023-01.md index 3466611c6..00da2d24f 100644 --- a/content/posts/2023-01.md +++ b/content/posts/2023-01.md @@ -48,4 +48,31 @@ $ psql -c 'SELECT * FROM pg_locks pl LEFT JOIN pg_stat_activity psa ON pl.pid = - The current time on the server is 08:52 and I see the dspaceCli locks were started at 04:00 and 05:00... so I need to check which cron jobs those belong to as I think I noticed this last month too - I'm going to wait and see if they finish, but by tomorrow I will kill them +## 2023-01-02 + +- The load on the server is now very low and there are no more locks from dspaceCli + - So there *was* some long-running process that was running and had to finish! + - That finally sheds some light on the "high load on Sunday" problem where I couldn't find any other distinct pattern in the nginx or Tomcat requests + +## 2023-01-03 + +- The load from the server on Sundays, which I have noticed for a long time, seems to be coming from the DSpace checker cron job + - This checks the checksums of all bitstreams to see if they match the ones in the database +- I exported the entire CGSpace metadata to do country/region checks with `csv-metadata-quality` + - I extracted only the items with countries, which was about 48,000, then split the file into parts of 10,000 items, but the upload found 2,000 changes in the first one and took several hours to complete... +- IWMI sent me ORCID identifiers for new scientsts, bringing our total to 2,010 + +## 2023-01-04 + +- I finally finished applying the region imports (in five batches of 10,000) + - It was about 7,500 missing regions in total... +- Now I will move on to doing the Initiative mappings + - I modified my `fix-initiative-mappings.py` script to only write out the items that have updated mappings + - This makes it way easier to apply fixes to the entire CGSpace because we don't try to import 100,000 items with no changes in mappings +- More dspaceCli locks from 04:00 this morning (current time on server is 07:33) and today is a Wednesday + - The checker cron job runs on `0,3`, which is Sunday and Wednesday, so this is from that... + - Finally at 16:30 I decided to kill the PIDs associated with those locks... + - I am going to disable that cron job for now and watch the server load for a few weeks +- Start a harvest on AReS + diff --git a/docs/2022-11/index.html b/docs/2022-11/index.html index afcde1d32..09ffbe1d1 100644 --- a/docs/2022-11/index.html +++ b/docs/2022-11/index.html @@ -24,7 +24,7 @@ I reverted the Cocoon autosave change because it was more of a nuissance that Pe - + @@ -54,9 +54,9 @@ I reverted the Cocoon autosave change because it was more of a nuissance that Pe "@type": "BlogPosting", "headline": "November, 2022", "url": "https://alanorth.github.io/cgspace-notes/2022-11/", - "wordCount": "3414", + "wordCount": "3411", "datePublished": "2022-11-01T09:11:36+03:00", - "dateModified": "2022-12-03T10:46:29+03:00", + "dateModified": "2023-01-04T10:53:02+03:00", "author": { "@type": "Person", "name": "Alan Orth" @@ -721,7 +721,7 @@ I reverted the Cocoon autosave change because it was more of a nuissance that Pe 60 dspaceCli 176 dspaceApi 1194 dspaceWeb -

!PostgreSQL database locks

+

PostgreSQL database locks

+

2023-01-02

+ +

2023-01-03

+ +

2023-01-04

+ diff --git a/docs/categories/index.html b/docs/categories/index.html index e85aca135..f41003057 100644 --- a/docs/categories/index.html +++ b/docs/categories/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/categories/notes/index.html b/docs/categories/notes/index.html index 4a5bc5b7b..c3bbcb70b 100644 --- a/docs/categories/notes/index.html +++ b/docs/categories/notes/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/categories/notes/page/2/index.html b/docs/categories/notes/page/2/index.html index 1422c1f30..fb66b18b4 100644 --- a/docs/categories/notes/page/2/index.html +++ b/docs/categories/notes/page/2/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/categories/notes/page/3/index.html b/docs/categories/notes/page/3/index.html index f91581675..568397a84 100644 --- a/docs/categories/notes/page/3/index.html +++ b/docs/categories/notes/page/3/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/categories/notes/page/4/index.html b/docs/categories/notes/page/4/index.html index 181271d27..18ef3e30f 100644 --- a/docs/categories/notes/page/4/index.html +++ b/docs/categories/notes/page/4/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/categories/notes/page/5/index.html b/docs/categories/notes/page/5/index.html index b9aa6f376..1ff2adf58 100644 --- a/docs/categories/notes/page/5/index.html +++ b/docs/categories/notes/page/5/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/categories/notes/page/6/index.html b/docs/categories/notes/page/6/index.html index 14002e90e..d23950994 100644 --- a/docs/categories/notes/page/6/index.html +++ b/docs/categories/notes/page/6/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/categories/notes/page/7/index.html b/docs/categories/notes/page/7/index.html index 70142088a..ccb448cd6 100644 --- a/docs/categories/notes/page/7/index.html +++ b/docs/categories/notes/page/7/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/index.html b/docs/index.html index 6095417d6..db3c5e93d 100644 --- a/docs/index.html +++ b/docs/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/page/2/index.html b/docs/page/2/index.html index f3a3da9f1..a6df5a1d3 100644 --- a/docs/page/2/index.html +++ b/docs/page/2/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/page/3/index.html b/docs/page/3/index.html index 235be3721..bf917b8a2 100644 --- a/docs/page/3/index.html +++ b/docs/page/3/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/page/4/index.html b/docs/page/4/index.html index 03e970e06..9c2e8fecc 100644 --- a/docs/page/4/index.html +++ b/docs/page/4/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/page/5/index.html b/docs/page/5/index.html index 706dc29d3..fc635a682 100644 --- a/docs/page/5/index.html +++ b/docs/page/5/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/page/6/index.html b/docs/page/6/index.html index 3c1a689f1..257858760 100644 --- a/docs/page/6/index.html +++ b/docs/page/6/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/page/7/index.html b/docs/page/7/index.html index ddf0ae804..82e516dc3 100644 --- a/docs/page/7/index.html +++ b/docs/page/7/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/page/8/index.html b/docs/page/8/index.html index 97946b818..f27a6ff61 100644 --- a/docs/page/8/index.html +++ b/docs/page/8/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/page/9/index.html b/docs/page/9/index.html index ea174da82..f9c8ee15c 100644 --- a/docs/page/9/index.html +++ b/docs/page/9/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/posts/index.html b/docs/posts/index.html index 928fc9090..4fb229939 100644 --- a/docs/posts/index.html +++ b/docs/posts/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/posts/page/2/index.html b/docs/posts/page/2/index.html index 8c8115fd2..ccc575711 100644 --- a/docs/posts/page/2/index.html +++ b/docs/posts/page/2/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/posts/page/3/index.html b/docs/posts/page/3/index.html index eeeee3f91..babb03ac0 100644 --- a/docs/posts/page/3/index.html +++ b/docs/posts/page/3/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/posts/page/4/index.html b/docs/posts/page/4/index.html index f7142f72e..822a0fc78 100644 --- a/docs/posts/page/4/index.html +++ b/docs/posts/page/4/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/posts/page/5/index.html b/docs/posts/page/5/index.html index 17835c33f..84de10ec7 100644 --- a/docs/posts/page/5/index.html +++ b/docs/posts/page/5/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/posts/page/6/index.html b/docs/posts/page/6/index.html index fd02bdf26..208259b92 100644 --- a/docs/posts/page/6/index.html +++ b/docs/posts/page/6/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/posts/page/7/index.html b/docs/posts/page/7/index.html index 1b63f8b25..bd5533aac 100644 --- a/docs/posts/page/7/index.html +++ b/docs/posts/page/7/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/posts/page/8/index.html b/docs/posts/page/8/index.html index 94de3e1e9..7e96f6dd1 100644 --- a/docs/posts/page/8/index.html +++ b/docs/posts/page/8/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/posts/page/9/index.html b/docs/posts/page/9/index.html index 2770568e0..2becb6a8b 100644 --- a/docs/posts/page/9/index.html +++ b/docs/posts/page/9/index.html @@ -10,7 +10,7 @@ - + diff --git a/docs/sitemap.xml b/docs/sitemap.xml index bdb39a7a6..663df202f 100644 --- a/docs/sitemap.xml +++ b/docs/sitemap.xml @@ -3,25 +3,25 @@ xmlns:xhtml="http://www.w3.org/1999/xhtml"> https://alanorth.github.io/cgspace-notes/categories/ - 2023-01-01T08:44:36+03:00 + 2023-01-04T10:53:02+03:00 https://alanorth.github.io/cgspace-notes/ - 2023-01-01T08:44:36+03:00 + 2023-01-04T10:53:02+03:00 https://alanorth.github.io/cgspace-notes/2023-01/ - 2023-01-01T08:44:36+03:00 + 2023-01-01T10:12:13+02:00 https://alanorth.github.io/cgspace-notes/categories/notes/ - 2023-01-01T08:44:36+03:00 + 2023-01-04T10:53:02+03:00 https://alanorth.github.io/cgspace-notes/posts/ - 2023-01-01T08:44:36+03:00 + 2023-01-04T10:53:02+03:00 https://alanorth.github.io/cgspace-notes/2022-12/ - 2022-12-29T08:32:08+02:00 + 2023-01-01T10:12:13+02:00 https://alanorth.github.io/cgspace-notes/2022-11/ - 2022-12-03T10:46:29+03:00 + 2023-01-04T10:53:02+03:00 https://alanorth.github.io/cgspace-notes/2022-10/ 2022-10-31T16:59:47+03:00