diff --git a/content/2016-04.md b/content/2016-04.md index 4619b66cf..7a7ecb41c 100644 --- a/content/2016-04.md +++ b/content/2016-04.md @@ -324,3 +324,9 @@ $ grep -c "Aborting context in finally statement" dspace.log.2016-04-20 - Test embargo during item upload - Seems to be working but the help text is misleading as to the date format +- It turns out the `robots.txt` issue we thought we solved last month isn't solved because you can't use wildcards in URL patterns: https://jira.duraspace.org/browse/DS-2962 +- Write some nginx rules to add `X-Robots-Tag` HTTP headers to the dynamic requests from `robots.txt` instead +- A few URLs to test with: + - https://dspacetest.cgiar.org/handle/10568/440/browse?type=bioversity + - https://dspacetest.cgiar.org/handle/10568/913/discover + - https://dspacetest.cgiar.org/handle/10568/1/search-filter?filtertype_0=country&filter_0=VIETNAM&filter_relational_operator_0=equals&field=country diff --git a/public/2016-04/index.html b/public/2016-04/index.html index fa630d55e..455722bba 100644 --- a/public/2016-04/index.html +++ b/public/2016-04/index.html @@ -449,6 +449,15 @@ UPDATE 46075 diff --git a/public/index.xml b/public/index.xml index b05c57356..c089aeb13 100644 --- a/public/index.xml +++ b/public/index.xml @@ -387,6 +387,15 @@ UPDATE 46075 <ul> <li>Test embargo during item upload</li> <li>Seems to be working but the help text is misleading as to the date format</li> +<li>It turns out the <code>robots.txt</code> issue we thought we solved last month isn&rsquo;t solved because you can&rsquo;t use wildcards in URL patterns: <a href="https://jira.duraspace.org/browse/DS-2962">https://jira.duraspace.org/browse/DS-2962</a></li> +<li>Write some nginx rules to add <code>X-Robots-Tag</code> HTTP headers to the dynamic requests from <code>robots.txt</code> instead</li> +<li>A few URLs to test with: + +<ul> +<li><a href="https://dspacetest.cgiar.org/handle/10568/440/browse?type=bioversity">https://dspacetest.cgiar.org/handle/10568/440/browse?type=bioversity</a></li> +<li><a href="https://dspacetest.cgiar.org/handle/10568/913/discover">https://dspacetest.cgiar.org/handle/10568/913/discover</a></li> +<li><a href="https://dspacetest.cgiar.org/handle/10568/1/search-filter?filtertype_0=country&amp;filter_0=VIETNAM&amp;filter_relational_operator_0=equals&amp;field=country">https://dspacetest.cgiar.org/handle/10568/1/search-filter?filtertype_0=country&amp;filter_0=VIETNAM&amp;filter_relational_operator_0=equals&amp;field=country</a></li> +</ul></li> </ul> diff --git a/public/tags/notes/index.xml b/public/tags/notes/index.xml index b5c1cc9d1..388d7a3e0 100644 --- a/public/tags/notes/index.xml +++ b/public/tags/notes/index.xml @@ -387,6 +387,15 @@ UPDATE 46075 <ul> <li>Test embargo during item upload</li> <li>Seems to be working but the help text is misleading as to the date format</li> +<li>It turns out the <code>robots.txt</code> issue we thought we solved last month isn&rsquo;t solved because you can&rsquo;t use wildcards in URL patterns: <a href="https://jira.duraspace.org/browse/DS-2962">https://jira.duraspace.org/browse/DS-2962</a></li> +<li>Write some nginx rules to add <code>X-Robots-Tag</code> HTTP headers to the dynamic requests from <code>robots.txt</code> instead</li> +<li>A few URLs to test with: + +<ul> +<li><a href="https://dspacetest.cgiar.org/handle/10568/440/browse?type=bioversity">https://dspacetest.cgiar.org/handle/10568/440/browse?type=bioversity</a></li> +<li><a href="https://dspacetest.cgiar.org/handle/10568/913/discover">https://dspacetest.cgiar.org/handle/10568/913/discover</a></li> +<li><a href="https://dspacetest.cgiar.org/handle/10568/1/search-filter?filtertype_0=country&amp;filter_0=VIETNAM&amp;filter_relational_operator_0=equals&amp;field=country">https://dspacetest.cgiar.org/handle/10568/1/search-filter?filtertype_0=country&amp;filter_0=VIETNAM&amp;filter_relational_operator_0=equals&amp;field=country</a></li> +</ul></li> </ul>