Version 5.3

Make sure we don't modify thumbnails if the item is an Infographic
because the JPG in the ORIGINAL bundle might actually be the "real"
file, in which case the THUMBNAIL bundle would have a legitimate
".jpg.jpg" file.

Also, limit the criteria for replacement to original bitstreams
that are less than 100KiB. In my tests I found that we had 4,022
items with ".jpg.jpg" thumbnails, and the average file size of the
originals in those items was 98KiB. Without considering the large
inforgraphics, which are several megabytes apiece, the average of
the remaining 3,765 originals was ~20KiB so 100KiB should be very
safe.

README.md

@@ -1,7 +1,8 @@
-# DSpace Curation Tasks [![Build Status](https://travis-ci.org/ilri/dspace-curation-tasks.svg?branch=master)](https://travis-ci.org/ilri/dspace-curation-tasks)
-Metadata curation tasks used on the [CGSpace](https://cgspace.cgiar.org) institutional repository:
+# CGSpace Java Helpers [![Build Status](https://travis-ci.org/ilri/cgspace-java-helpers.svg?branch=dspace5)](https://travis-ci.org/ilri/dspace-curation-tasks)
+DSpace curation tasks and other Java-based helpers used on the [CGSpace](https://cgspace.cgiar.org) institutional repository:
 
 - **CountryCodeTagger**: add ISO 3166-1 Alpha2 country codes to items based on their existing country metadata
+- **FixJpgJpgThumbnails**: Fix low-quality ".jpg.jpg" thumbnails by replacing them with their originals
 
 Tested on DSpace 5.8. Read more about the [DSpace curation system](https://wiki.lyrasis.org/display/DSDOC5x/Curation+System).
 
@@ -13,8 +14,8 @@ To use these curation tasks in a DSpace project add the following dependency to
 ```
 <dependency>
   <groupId>io.github.ilri.cgspace</groupId>
-  <artifactId>dspace-curation-tasks</artifactId>
-  <version>1.0-SNAPSHOT</version>
+  <artifactId>cgspace-java-helpers</artifactId>
+  <version>5.3</version>
 </dependency>
 ```
 
@@ -30,7 +31,7 @@ $ mvn package
 Copy the resulting jar to the DSpace `lib` directory:
 
 ```
-$ cp target/dspace-curation-tasks-1.0-SNAPSHOT.jar ~/dspace/lib/dspace-curation-tasks-1.0-SNAPSHOT.jar
+$ cp target/cgspace-java-helpers-5.3.jar ~/dspace/lib
 ```
 
 ## Configuration
@@ -71,14 +72,12 @@ $ ~/dspace/bin/dspace curate -t countrycodetagger -i 10568/3 -r - -l 500 -s obje
 This project was initially created according to the [Maven Getting Started Guide](https://maven.apache.org/guides/getting-started/):
 
 ```console
-$ mvn -B archetype:generate -DgroupId=io.github.ilri.cgspace -DartifactId=dspace-curation-tasks -DarchetypeArtifactId=maven-archetype-quickstart -DarchetypeVersion=1.4
+$ mvn -B archetype:generate -DgroupId=io.github.ilri.cgspace -DartifactId=cgspace-java-helpers -DarchetypeArtifactId=maven-archetype-quickstart -DarchetypeVersion=1.4
 ```
 
 ## TODO
 
 - Make sure this doesn't work on items in the workflow
-- Port to DSpace 6
-  - Remember to bump Gson version!
 - Check for existence of metadata field before trying to add metadata
 - Add tests
 

pom.xml

@@ -5,11 +5,11 @@
   <modelVersion>4.0.0</modelVersion>
 
   <groupId>io.github.ilri.cgspace</groupId>
-  <artifactId>dspace-curation-tasks</artifactId>
-  <version>1.0-SNAPSHOT</version>
+  <artifactId>cgspace-java-helpers</artifactId>
+  <version>5.3</version>
 
-  <name>dspace-curation-tasks</name>
-  <url>https://github.com/ilri/dspace-curation-tasks</url>
+  <name>cgspace-java-helpers</name>
+  <url>https://github.com/ilri/cgspace-java-helpers</url>
 
   <licenses>
     <license>
@@ -53,9 +53,9 @@
   </dependencies>
 
   <scm>
-      <connection>scm:git:git://github.com/ilri/dspace-curation-tasks.git</connection>
-      <developerConnection>scm:git:ssh://github.com:nanosai/dspace-curation-tasks.git</developerConnection>
-      <url>http://github.com/ilri/dspace-curation-tasks</url>
+      <connection>scm:git:git://github.com/ilri/cgspace-java-helpers.git</connection>
+      <developerConnection>scm:git:ssh://github.com:nanosai/cgspace-java-helpers.git</developerConnection>
+      <url>http://github.com/ilri/cgspace-java-helpers</url>
   </scm>
 
   <distributionManagement>

src/main/java/io/github/ilri/cgspace/scripts/FixJpgJpgThumbnails.java

@@ -0,0 +1,131 @@
+package io.github.ilri.cgspace.scripts;
+
+import org.apache.commons.lang.StringUtils;
+import org.dspace.authorize.AuthorizeException;
+import org.dspace.content.*;
+import org.dspace.core.Constants;
+import org.dspace.core.Context;
+import org.dspace.handle.HandleManager;
+
+import java.io.IOException;
+import java.sql.SQLException;
+
+/**
+ * @author Andrea Schweer schweer@waikato.ac.nz for the LCoNZ Institutional Research Repositories
+ * @author Alan Orth for the International Livestock Research Institute
+ * @version 5.3
+ * @since 5.1
+ */
+public class FixJpgJpgThumbnails {
+
+	public static void main(String[] args) {
+		String parentHandle = null;
+		if (args.length >= 1) {
+			parentHandle = args[0];
+		}
+
+		Context context = null;
+		try {
+			context = new Context();
+			context.turnOffAuthorisationSystem();
+
+			if (StringUtils.isBlank(parentHandle)) {
+				process(context, Item.findAll(context));
+			} else {
+				DSpaceObject parent = HandleManager.resolveToObject(context, parentHandle);
+				if (parent != null) {
+					switch (parent.getType()) {
+						case Constants.COLLECTION:
+							process(context, ((Collection) parent).getAllItems()); // getAllItems because we want to work on non-archived ones as well
+							break;
+						case Constants.COMMUNITY:
+							Collection[] collections = ((Community) parent).getCollections();
+							for (Collection collection : collections) {
+								process(context, collection.getAllItems()); // getAllItems because we want to work on non-archived ones as well
+							}
+							break;
+						case Constants.SITE:
+							process(context, Item.findAll(context));
+							break;
+						case Constants.ITEM:
+							processItem((Item) parent);
+							context.commit();
+							break;
+					}
+				}
+			}
+		} catch (SQLException | AuthorizeException | IOException e) {
+			e.printStackTrace(System.err);
+		} finally {
+			if (context != null && context.isValid()) {
+				context.abort();
+			}
+		}
+	}
+
+	private static void process(Context context, ItemIterator items) throws SQLException, IOException, AuthorizeException {
+		while (items.hasNext()) {
+			Item item = items.next();
+			processItem(item);
+			context.commit();
+			item.decache();
+		}
+	}
+
+	private static void processItem(Item item) throws SQLException, AuthorizeException, IOException {
+		// Some bitstreams like Infographics are large JPGs and put in the ORIGINAL bundle on purpose so we shouldn't
+		// swap them.
+		Metadatum[] itemTypes = item.getMetadataByMetadataString("dc.type");
+		Boolean itemHasInfographic = false;
+		for (Metadatum itemType: itemTypes) {
+			if (itemType.value.equals("Infographic")) {
+				itemHasInfographic = true;
+			}
+		}
+
+		Bundle[] thumbnailBundles = item.getBundles("THUMBNAIL");
+		for (Bundle thumbnailBundle : thumbnailBundles) {
+			Bitstream[] thumbnailBundleBitstreams = thumbnailBundle.getBitstreams();
+			for (Bitstream thumbnailBitstream : thumbnailBundleBitstreams) {
+				String thumbnailName = thumbnailBitstream.getName();
+
+				if (thumbnailName.toLowerCase().contains(".jpg.jpg")) {
+					Bundle[] originalBundles = item.getBundles("ORIGINAL");
+					for (Bundle originalBundle : originalBundles) {
+						Bitstream[] originalBundleBitstreams = originalBundle.getBitstreams();
+
+						for (Bitstream originalBitstream : originalBundleBitstreams) {
+							String originalName = originalBitstream.getName();
+
+							Long originalBitstreamBytes = originalBitstream.getSize();
+
+							/*
+							- check if the original file name is the same as the thumbnail name minus the extra ".jpg"
+							- check if the thumbnail description indicates it was automatically generated
+							- check if the item has dc.type Infographic (JPG could be the "real" item!)
+							- check if the original bitstream is less than ~100KiB
+							    - Note: in my tests there were 4022 items with ".jpg.jpg" thumbnails totaling 394549249
+							      bytes for an average of about 98KiB so ~100KiB seems like a good cut off
+							*/
+							if (
+									originalName.equalsIgnoreCase(StringUtils.removeEndIgnoreCase(thumbnailName, ".jpg"))
+									&& ("Generated Thumbnail".equals(thumbnailBitstream.getDescription()) || "IM Thumbnail".equals(thumbnailBitstream.getDescription()))
+									&& !itemHasInfographic
+									&& originalBitstreamBytes < 100000
+							) {
+								System.out.println(item.getHandle() + ": replacing " + thumbnailName + " with " + originalName);
+
+								//add the original bitstream to the THUMBNAIL bundle
+								thumbnailBundle.addBitstream(originalBitstream);
+								//remove the original bitstream from the ORIGINAL bundle
+								originalBundle.removeBitstream(originalBitstream);
+								//remove the JpgJpg bitstream from the THUMBNAIL bundle
+								thumbnailBundle.removeBitstream(thumbnailBitstream);
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+}