From e02d82d56bf8886451020958c0aeddeb6b440ea0 Mon Sep 17 00:00:00 2001 From: Alan Orth Date: Wed, 24 Jan 2018 17:12:47 +0200 Subject: [PATCH] Add notes for 2018-01-24 --- content/post/2018-01.md | 67 ++++++++++++++++++++++++++++++++++ public/2018-01/index.html | 76 +++++++++++++++++++++++++++++++++++++-- public/sitemap.xml | 10 +++--- 3 files changed, 145 insertions(+), 8 deletions(-) diff --git a/content/post/2018-01.md b/content/post/2018-01.md index 5617ebeef..e9d8fcdf6 100644 --- a/content/post/2018-01.md +++ b/content/post/2018-01.md @@ -1047,3 +1047,70 @@ $ ./rest-find-collections.py 10568/1 | grep -i untitled ``` - I can definitely design a test plan on this! + +## 2018-01-24 + +- Looking at the REST requests, most of them are to expand all or metadata, but 5% are for retrieving bitstreams: + +``` +# zcat --force /var/log/nginx/access.log.3.gz /var/log/nginx/access.log.4.gz /var/log/nginx/library-access.log.3.gz /var/log/nginx/library-access.log.4.gz /var/log/nginx/rest.log.3.gz /var/log/nginx/rest.log.4.gz /var/log/nginx/oai.log.3.gz /var/log/nginx/oai.log.4.gz /var/log/nginx/error.log.3.gz /var/log/nginx/error.log.4.gz | grep "21/Jan/2018" | grep "GET " | grep -v "/admin" | awk '{print $7}' | grep -E "^/rest" | grep -Eo "(retrieve|expand=[a-z].*)" | sort | uniq -c | sort -n + 1 expand=collections + 16 expand=all&limit=1 + 45 expand=items + 775 retrieve + 5675 expand=all + 8633 expand=metadata +``` + +- I finished creating the test plan for DSpace Test and ran it from my Linode with: + +``` +$ jmeter -n -t DSpacePerfTest-dspacetest.cgiar.org.jmx -l 2018-01-24-1.jtl +``` + +- Atmire responded to [my issue from two weeks ago](https://tracker.atmire.com/tickets-cgiar-ilri/view-ticket?id=560) and said they will start looking into DSpace 5.8 compatibility for CGSpace +- I set up a new Arch Linux Linode instance with 8192 MB of RAM and ran the test plan a few times to get a baseline: + +``` +# lscpu +# lscpu +Architecture: x86_64 +CPU op-mode(s): 32-bit, 64-bit +Byte Order: Little Endian +CPU(s): 4 +On-line CPU(s) list: 0-3 +Thread(s) per core: 1 +Core(s) per socket: 1 +Socket(s): 4 +NUMA node(s): 1 +Vendor ID: GenuineIntel +CPU family: 6 +Model: 63 +Model name: Intel(R) Xeon(R) CPU E5-2680 v3 @ 2.50GHz +Stepping: 2 +CPU MHz: 2499.994 +BogoMIPS: 5001.32 +Hypervisor vendor: KVM +Virtualization type: full +L1d cache: 32K +L1i cache: 32K +L2 cache: 4096K +L3 cache: 16384K +NUMA node0 CPU(s): 0-3 +Flags: fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon rep_good nopl xtopology cpuid pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm cpuid_fault invpcid_single pti retpoline fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid xsaveopt arat +# free -m + total used free shared buff/cache available +Mem: 7970 107 7759 1 103 7771 +Swap: 255 0 255 +# pacman -Syu +# pacman -S git wget jre8-openjdk-headless mosh htop tmux +# useradd -m test +# su - test +$ git clone -b ilri https://github.com/alanorth/dspace-performance-test.git +$ wget http://www-us.apache.org/dist//jmeter/binaries/apache-jmeter-3.3.tgz +$ tar xf apache-jmeter-3.3.tgz +$ cd apache-jmeter-3.3/bin +$ ./jmeter -n -t ~/dspace-performance-test/DSpacePerfTest-dspacetest.cgiar.org.jmx -l ~/dspace-performance-test/2018-01-24-linode5451120-baseline.jtl -j ~/dspace-performance-test/2018-01-24-linode5451120-baseline.log +$ ./jmeter -n -t ~/dspace-performance-test/DSpacePerfTest-dspacetest.cgiar.org.jmx -l ~/dspace-performance-test/2018-01-24-linode5451120-baseline2.jtl -j ~/dspace-performance-test/2018-01-24-linode5451120-baseline2.log +$ ./jmeter -n -t ~/dspace-performance-test/DSpacePerfTest-dspacetest.cgiar.org.jmx -l ~/dspace-performance-test/2018-01-24-linode5451120-baseline3.jtl -j ~/dspace-performance-test/2018-01-24-linode5451120-baseline3.log +``` diff --git a/public/2018-01/index.html b/public/2018-01/index.html index 169934994..4006dca68 100644 --- a/public/2018-01/index.html +++ b/public/2018-01/index.html @@ -92,7 +92,7 @@ Danny wrote to ask for help renewing the wildcard ilri.org certificate and I adv - + @@ -194,9 +194,9 @@ Danny wrote to ask for help renewing the wildcard ilri.org certificate and I adv "@type": "BlogPosting", "headline": "January, 2018", "url": "https://alanorth.github.io/cgspace-notes/2018-01/", - "wordCount": "5977", + "wordCount": "6355", "datePublished": "2018-01-02T08:35:54-08:00", - "dateModified": "2018-01-23T00:08:32+02:00", + "dateModified": "2018-01-23T18:08:57+02:00", "author": { "@type": "Person", "name": "Alan Orth" @@ -1418,6 +1418,76 @@ $ ./rest-find-collections.py 10568/1 | grep -i untitled
  • I can definitely design a test plan on this!
    • +

    2018-01-24

    + + + +
    # zcat --force /var/log/nginx/access.log.3.gz /var/log/nginx/access.log.4.gz /var/log/nginx/library-access.log.3.gz /var/log/nginx/library-access.log.4.gz /var/log/nginx/rest.log.3.gz /var/log/nginx/rest.log.4.gz /var/log/nginx/oai.log.3.gz /var/log/nginx/oai.log.4.gz /var/log/nginx/error.log.3.gz /var/log/nginx/error.log.4.gz | grep "21/Jan/2018" | grep "GET " | grep -v "/admin" | awk '{print $7}' | grep -E "^/rest" | grep -Eo "(retrieve|expand=[a-z].*)" | sort | uniq -c | sort -n
+      1 expand=collections
+     16 expand=all&limit=1
+     45 expand=items
+    775 retrieve
+   5675 expand=all
+   8633 expand=metadata
+
    + + + +
    $ jmeter -n -t DSpacePerfTest-dspacetest.cgiar.org.jmx -l 2018-01-24-1.jtl
+
    + + + +
    # lscpu
+# lscpu 
+Architecture:        x86_64
+CPU op-mode(s):      32-bit, 64-bit
+Byte Order:          Little Endian
+CPU(s):              4
+On-line CPU(s) list: 0-3
+Thread(s) per core:  1
+Core(s) per socket:  1
+Socket(s):           4
+NUMA node(s):        1
+Vendor ID:           GenuineIntel
+CPU family:          6
+Model:               63
+Model name:          Intel(R) Xeon(R) CPU E5-2680 v3 @ 2.50GHz
+Stepping:            2
+CPU MHz:             2499.994
+BogoMIPS:            5001.32
+Hypervisor vendor:   KVM
+Virtualization type: full
+L1d cache:           32K
+L1i cache:           32K
+L2 cache:            4096K
+L3 cache:            16384K
+NUMA node0 CPU(s):   0-3
+Flags:               fpu vme de pse tsc msr pae mce cx8 apic sep mtrr pge mca cmov pat pse36 clflush mmx fxsr sse sse2 ss syscall nx pdpe1gb rdtscp lm constant_tsc arch_perfmon rep_good nopl xtopology cpuid pni pclmulqdq ssse3 fma cx16 pcid sse4_1 sse4_2 x2apic movbe popcnt tsc_deadline_timer aes xsave avx f16c rdrand hypervisor lahf_lm abm cpuid_fault invpcid_single pti retpoline fsgsbase tsc_adjust bmi1 avx2 smep bmi2 erms invpcid xsaveopt arat
+# free -m
+              total        used        free      shared  buff/cache   available
+Mem:           7970         107        7759           1         103        7771
+Swap:           255           0         255
+# pacman -Syu
+# pacman -S git wget jre8-openjdk-headless mosh htop tmux
+# useradd -m test
+# su - test
+$ git clone -b ilri https://github.com/alanorth/dspace-performance-test.git
+$ wget http://www-us.apache.org/dist//jmeter/binaries/apache-jmeter-3.3.tgz
+$ tar xf apache-jmeter-3.3.tgz
+$ cd apache-jmeter-3.3/bin
+$ ./jmeter -n -t ~/dspace-performance-test/DSpacePerfTest-dspacetest.cgiar.org.jmx -l ~/dspace-performance-test/2018-01-24-linode5451120-baseline.jtl -j ~/dspace-performance-test/2018-01-24-linode5451120-baseline.log
+$ ./jmeter -n -t ~/dspace-performance-test/DSpacePerfTest-dspacetest.cgiar.org.jmx -l ~/dspace-performance-test/2018-01-24-linode5451120-baseline2.jtl -j ~/dspace-performance-test/2018-01-24-linode5451120-baseline2.log
+$ ./jmeter -n -t ~/dspace-performance-test/DSpacePerfTest-dspacetest.cgiar.org.jmx -l ~/dspace-performance-test/2018-01-24-linode5451120-baseline3.jtl -j ~/dspace-performance-test/2018-01-24-linode5451120-baseline3.log
+
    + diff --git a/public/sitemap.xml b/public/sitemap.xml index d037301a4..69d728ae8 100644 --- a/public/sitemap.xml +++ b/public/sitemap.xml @@ -4,7 +4,7 @@ https://alanorth.github.io/cgspace-notes/2018-01/ - 2018-01-23T00:08:32+02:00 + 2018-01-23T18:08:57+02:00 @@ -144,7 +144,7 @@ https://alanorth.github.io/cgspace-notes/ - 2018-01-23T00:08:32+02:00 + 2018-01-23T18:08:57+02:00 0 @@ -155,7 +155,7 @@ https://alanorth.github.io/cgspace-notes/tags/notes/ - 2018-01-23T00:08:32+02:00 + 2018-01-23T18:08:57+02:00 0 @@ -167,13 +167,13 @@ https://alanorth.github.io/cgspace-notes/post/ - 2018-01-23T00:08:32+02:00 + 2018-01-23T18:08:57+02:00 0 https://alanorth.github.io/cgspace-notes/tags/ - 2018-01-23T00:08:32+02:00 + 2018-01-23T18:08:57+02:00 0