alanorth/cgspace-java-helpers
1
0
Fork 0
mirror of https://github.com/ilri/cgspace-java-helpers.git synced 2024-11-28 09:38:21 +01:00
Code Issues Projects Releases Wiki Activity

Compare commits

base: alanorth:b396fba04342d5cb416e412692f2d5ca9358edc8
Branches Tags
alanorth:dspace7
alanorth:dspace6
alanorth:dspace5
alanorth:master
alanorth:v7.6.1.3
alanorth:v7.6.1.2
alanorth:v7.6.1.1
alanorth:v7.6.1
alanorth:v7.6
alanorth:v6.2
alanorth:v6.1
alanorth:v6.0
alanorth:v5.3
alanorth:v5.2
alanorth:v5.1
..
compare: alanorth:f0754ab4193dc3809bb0210de2c38188814b563a
Branches Tags
alanorth:dspace7
alanorth:dspace6
alanorth:dspace5
alanorth:master
alanorth:v7.6.1.3
alanorth:v7.6.1.2
alanorth:v7.6.1.1
alanorth:v7.6.1
alanorth:v7.6
alanorth:v6.2
alanorth:v6.1
alanorth:v6.0
alanorth:v5.3
alanorth:v5.2
alanorth:v5.1

No commits in common. "b396fba04342d5cb416e412692f2d5ca9358edc8" and "f0754ab4193dc3809bb0210de2c38188814b563a" have entirely different histories.
b396fba043 ... f0754ab419

6 changed files with 178 additions and 252 deletions
Show Stats Expand all files Collapse all files

7
src/main/java/io/github/ilri/cgspace/ctasks/CountriesVocabulary.java
View File

@ -19,8 +19,7 @@ public class CountriesVocabulary {
private String alpha_2; //required
private String alpha_3; //required
public Country(
String name,
public Country(String name,
@Nullable String common_name,
@Nullable String official_name,
@Nullable String cgspace_name,
@ -31,9 +30,7 @@ public class CountriesVocabulary {
this.common_name = common_name;
this.official_name = official_name;
this.cgspace_name = cgspace_name;
this.numeric =
numeric; // fuuuuu this is a string and we can't cast to Integer because some
// values are zeropadded like "004"
this.numeric = numeric; // fuuuuu this is a string and we can't cast to Integer because some values are zeropadded like "004"
this.alpha_2 = alpha_2;
this.alpha_3 = alpha_3;
}

76
src/main/java/io/github/ilri/cgspace/ctasks/CountryCodeTagger.java
View File

@ -7,7 +7,6 @@
package io.github.ilri.cgspace.ctasks;
import com.google.gson.Gson;
import org.apache.log4j.Logger;
import org.dspace.authorize.AuthorizeException;
import org.dspace.content.DSpaceObject;
@ -24,11 +23,11 @@ import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
public class CountryCodeTagger extends AbstractCurationTask {
public class CountryCodeTagger extends AbstractCurationTask
{
public class CountryCodeTaggerConfig {
private final String isocodesJsonPath = "/io/github/ilri/cgspace/ctasks/iso_3166-1.json";
private final String cgspaceCountriesJsonPath =
"/io/github/ilri/cgspace/ctasks/cgspace-countries.json";
private final String cgspaceCountriesJsonPath = "/io/github/ilri/cgspace/ctasks/cgspace-countries.json";
private final String iso3166Field = taskProperty("iso3166.field");
private final String iso3166Alpha2Field = taskProperty("iso3166-alpha2.field");
private final boolean forceupdate = taskBooleanProperty("forceupdate", false);
@ -58,11 +57,13 @@ public class CountryCodeTagger extends AbstractCurationTask {
}
@Override
public int perform(DSpaceObject dso) throws IOException {
public int perform(DSpaceObject dso) throws IOException
{
// gotta define this here so we can access it after the if context...
CountryCodeTaggerResult alpha2Result = new CountryCodeTaggerResult();
if (dso.getType() == Constants.ITEM) {
if (dso.getType() == Constants.ITEM)
{
// Load configuration
CountryCodeTaggerConfig config = new CountryCodeTaggerConfig();
@ -81,13 +82,11 @@ public class CountryCodeTagger extends AbstractCurationTask {
return alpha2Result.getStatus();
}
public CountryCodeTaggerResult performAlpha2(Item item, CountryCodeTaggerConfig config)
throws IOException, SQLException {
public CountryCodeTaggerResult performAlpha2(Item item, CountryCodeTaggerConfig config) throws IOException, SQLException {
CountryCodeTaggerResult alpha2Result = new CountryCodeTaggerResult();
String itemHandle = item.getHandle();
List<MetadataValue> itemCountries =
itemService.getMetadataByMetadataString(item, config.iso3166Field);
List<MetadataValue> itemCountries = itemService.getMetadataByMetadataString(item, config.iso3166Field);
// skip items that don't have country metadata
if (itemCountries.size() == 0) {
@ -96,55 +95,31 @@ public class CountryCodeTagger extends AbstractCurationTask {
} else {
Gson gson = new Gson();
// TODO: convert to try:
// https://docs.oracle.com/javase/tutorial/essential/exceptions/tryResourceClose.html
BufferedReader reader =
new BufferedReader(
new InputStreamReader(
this.getClass().getResourceAsStream(config.isocodesJsonPath)));
ISO3166CountriesVocabulary isocodesCountriesJson =
gson.fromJson(reader, ISO3166CountriesVocabulary.class);
// TODO: convert to try: https://docs.oracle.com/javase/tutorial/essential/exceptions/tryResourceClose.html
BufferedReader reader = new BufferedReader(new InputStreamReader(this.getClass().getResourceAsStream(config.isocodesJsonPath)));
ISO3166CountriesVocabulary isocodesCountriesJson = gson.fromJson(reader, ISO3166CountriesVocabulary.class);
reader.close();
reader =
new BufferedReader(
new InputStreamReader(
this.getClass()
.getResourceAsStream(config.cgspaceCountriesJsonPath)));
CGSpaceCountriesVocabulary cgspaceCountriesJson =
gson.fromJson(reader, CGSpaceCountriesVocabulary.class);
reader = new BufferedReader(new InputStreamReader(this.getClass().getResourceAsStream(config.cgspaceCountriesJsonPath)));
CGSpaceCountriesVocabulary cgspaceCountriesJson = gson.fromJson(reader, CGSpaceCountriesVocabulary.class);
reader.close();
// split the alpha2 country code field into schema, element, and qualifier so we can use
// it with item.addMetadata()
// split the alpha2 country code field into schema, element, and qualifier so we can use it with item.addMetadata()
String[] iso3166Alpha2FieldParts = config.iso3166Alpha2Field.split("\\.");
if (config.forceupdate) {
itemService.clearMetadata(
Curator.curationContext(),
item,
iso3166Alpha2FieldParts[0],
iso3166Alpha2FieldParts[1],
iso3166Alpha2FieldParts[2],
Item.ANY);
itemService.clearMetadata(Curator.curationContext(), item, iso3166Alpha2FieldParts[0], iso3166Alpha2FieldParts[1], iso3166Alpha2FieldParts[2], Item.ANY);
}
// check the item's country codes, if any
List<MetadataValue> itemAlpha2CountryCodes =
itemService.getMetadataByMetadataString(item, config.iso3166Alpha2Field);
List<MetadataValue> itemAlpha2CountryCodes = itemService.getMetadataByMetadataString(item, config.iso3166Alpha2Field);
if (itemAlpha2CountryCodes.size() == 0) {
List<String> newAlpha2Codes = new ArrayList<String>();
for (MetadataValue itemCountry : itemCountries) {
//check ISO 3166-1 countries
for (CountriesVocabulary.Country country : isocodesCountriesJson.countries) {
if (itemCountry.getValue().equalsIgnoreCase(country.getName())
|| itemCountry
.getValue()
.equalsIgnoreCase(country.get_official_name())
|| itemCountry
.getValue()
.equalsIgnoreCase(country.get_common_name())) {
if (itemCountry.getValue().equalsIgnoreCase(country.getName()) || itemCountry.getValue().equalsIgnoreCase(country.get_official_name()) || itemCountry.getValue().equalsIgnoreCase(country.get_common_name())) {
newAlpha2Codes.add(country.getAlpha_2());
}
}
@ -159,14 +134,7 @@ public class CountryCodeTagger extends AbstractCurationTask {
if (newAlpha2Codes.size() > 0) {
try {
itemService.addMetadata(
Curator.curationContext(),
item,
iso3166Alpha2FieldParts[0],
iso3166Alpha2FieldParts[1],
iso3166Alpha2FieldParts[2],
"en_US",
newAlpha2Codes);
itemService.addMetadata(Curator.curationContext(), item, iso3166Alpha2FieldParts[0], iso3166Alpha2FieldParts[1], iso3166Alpha2FieldParts[2], "en_US", newAlpha2Codes);
itemService.update(Curator.curationContext(), item);
} catch (SQLException | AuthorizeException sqle) {
config.log.debug(sqle.getMessage());
@ -174,11 +142,7 @@ public class CountryCodeTagger extends AbstractCurationTask {
alpha2Result.setStatus(Curator.CURATE_ERROR);
}
alpha2Result.setResult(
itemHandle
+ ": added "
+ newAlpha2Codes.size()
+ " alpha2 country code(s)");
alpha2Result.setResult(itemHandle + ": added " + newAlpha2Codes.size() + " alpha2 country code(s)");
} else {
alpha2Result.setResult(itemHandle + ": no matching countries found");
}

7
src/main/java/io/github/ilri/cgspace/ctasks/ISO3166CountriesVocabulary.java
View File

@ -7,12 +7,9 @@
package io.github.ilri.cgspace.ctasks;
import com.google.gson.annotations.SerializedName;
import java.util.List;
public class ISO3166CountriesVocabulary extends CountriesVocabulary {
// support reading iso_3166-1.json from Debian's iso-codes package using SerializedName since
// our class needs to match the JSON exactly
@SerializedName("3166-1")
List<Country> countries;
// support reading iso_3166-1.json from Debian's iso-codes package using SerializedName since our class needs to match the JSON exactly
@SerializedName("3166-1") List<Country> countries;
}

75
src/main/java/io/github/ilri/cgspace/scripts/FixJpgJpgThumbnails.java
View File

@ -8,20 +8,14 @@ package io.github.ilri.cgspace.scripts;
import org.apache.commons.lang.StringUtils;
import org.dspace.authorize.AuthorizeException;
import org.dspace.content.Bitstream;
import org.dspace.content.Bundle;
import org.dspace.content.Collection;
import org.dspace.content.Community;
import org.dspace.content.DSpaceObject;
import org.dspace.content.Item;
import org.dspace.content.MetadataValue;
import org.dspace.content.factory.ContentServiceFactory;
import org.dspace.content.service.BundleService;
import org.dspace.content.service.ItemService;
import org.dspace.content.*;
import org.dspace.core.Constants;
import org.dspace.core.Context;
import org.dspace.content.factory.ContentServiceFactory;
import org.dspace.content.service.ItemService;
import org.dspace.handle.factory.HandleServiceFactory;
import org.dspace.handle.service.HandleService;
import org.dspace.content.service.BundleService;
import java.io.IOException;
import java.sql.SQLException;
@ -37,10 +31,8 @@ import java.util.List;
public class FixJpgJpgThumbnails {
//note: static members belong to the class itself, not any one instance
public static ItemService itemService = ContentServiceFactory.getInstance().getItemService();
public static HandleService handleService =
HandleServiceFactory.getInstance().getHandleService();
public static BundleService bundleService =
ContentServiceFactory.getInstance().getBundleService();
public static HandleService handleService = HandleServiceFactory.getInstance().getHandleService();
public static BundleService bundleService = ContentServiceFactory.getInstance().getBundleService();
public static void main(String[] args) {
String parentHandle = null;
@ -60,16 +52,12 @@ public class FixJpgJpgThumbnails {
if (parent != null) {
switch (parent.getType()) {
case Constants.COLLECTION:
process(
context,
itemService.findByCollection(context, (Collection) parent));
process(context, itemService.findByCollection(context, (Collection) parent));
break;
case Constants.COMMUNITY:
List<Collection> collections = ((Community) parent).getCollections();
for (Collection collection : collections) {
process(
context,
itemService.findAllByCollection(context, collection));
process(context, itemService.findAllByCollection(context, collection));
}
break;
case Constants.SITE:
@ -91,8 +79,7 @@ public class FixJpgJpgThumbnails {
}
}
private static void process(Context context, Iterator<Item> items)
throws SQLException, IOException, AuthorizeException {
private static void process(Context context, Iterator<Item> items) throws SQLException, IOException, AuthorizeException {
while (items.hasNext()) {
Item item = items.next();
processItem(context, item);
@ -100,18 +87,14 @@ public class FixJpgJpgThumbnails {
}
}
private static void processItem(Context context, Item item)
throws SQLException, AuthorizeException, IOException {
// Some bitstreams like Infographics and Maps are large JPEGs and put in the ORIGINAL bundle
// on purpose so we shouldn't
private static void processItem(Context context, Item item) throws SQLException, AuthorizeException, IOException {
// Some bitstreams like Infographics are large JPGs and put in the ORIGINAL bundle on purpose so we shouldn't
// swap them.
List<MetadataValue> itemTypes =
itemService.getMetadataByMetadataString(item, "dcterms.type");
List<MetadataValue> itemTypes = itemService.getMetadataByMetadataString(item, "dcterms.type");
boolean itemHasInfographic = false;
for (MetadataValue itemType: itemTypes) {
if (itemType.getValue().equals("Infographic") || itemType.getValue().equals("Map")) {
System.out.println(
item.getHandle() + ": item has an Infographic or Map, skipping.");
return;
if (itemType.getValue().equals("Infographic")) {
itemHasInfographic = true;
}
}
@ -120,12 +103,6 @@ public class FixJpgJpgThumbnails {
List<Bitstream> thumbnailBundleBitstreams = thumbnailBundle.getBitstreams();
for (Bitstream thumbnailBitstream : thumbnailBundleBitstreams) {
String thumbnailName = thumbnailBitstream.getName();
String thumbnailDescription = thumbnailBitstream.getDescription();
// There is no point continuing if the thumbnail's description is empty or null
if (StringUtils.isEmpty(thumbnailDescription)) {
continue;
}
if (thumbnailName.toLowerCase().contains(".jpg.jpg")) {
List<Bundle> originalBundles = item.getBundles("ORIGINAL");
@ -140,25 +117,21 @@ public class FixJpgJpgThumbnails {
/*
- check if the original file name is the same as the thumbnail name minus the extra ".jpg"
- check if the thumbnail description indicates it was automatically generated
- check if the item has dc.type Infographic (JPG could be the "real" item!)
- check if the original bitstream is less than ~100KiB
- Note: in my tests there were 4022 items with ".jpg.jpg" thumbnails totaling 394549249
bytes for an average of about 98KiB so ~100KiB seems like a good cut off
*/
if (originalName.equalsIgnoreCase(
StringUtils.removeEndIgnoreCase(thumbnailName, ".jpg"))
&& ("Generated Thumbnail".equals(thumbnailDescription)
|| "IM Thumbnail".equals(thumbnailDescription))
&& originalBitstreamBytes < 100000) {
System.out.println(
item.getHandle()
+ ": replacing "
+ thumbnailName
+ " with "
+ originalName);
if (
originalName.equalsIgnoreCase(StringUtils.removeEndIgnoreCase(thumbnailName, ".jpg"))
&& ("Generated Thumbnail".equals(thumbnailBitstream.getDescription()) || "IM Thumbnail".equals(thumbnailBitstream.getDescription()))
&& !itemHasInfographic
&& originalBitstreamBytes < 100000
) {
System.out.println(item.getHandle() + ": replacing " + thumbnailName + " with " + originalName);
//add the original bitstream to the THUMBNAIL bundle
bundleService.addBitstream(
context, thumbnailBundle, originalBitstream);
bundleService.addBitstream(context, thumbnailBundle, originalBitstream);
//remove the original bitstream from the ORIGINAL bundle
originalBundle.removeBitstream(originalBitstream);
//remove the JpgJpg bitstream from the THUMBNAIL bundle

7
src/main/java/io/github/ilri/cgspace/scripts/FixLowQualityThumbnails.java
View File

@ -8,12 +8,7 @@ package io.github.ilri.cgspace.scripts;
import org.apache.commons.lang.StringUtils;
import org.dspace.authorize.AuthorizeException;
import org.dspace.content.Bitstream;
import org.dspace.content.Bundle;
import org.dspace.content.Collection;
import org.dspace.content.Community;
import org.dspace.content.DSpaceObject;
import org.dspace.content.Item;
import org.dspace.content.*;
import org.dspace.content.factory.ContentServiceFactory;
import org.dspace.content.service.BundleService;
import org.dspace.content.service.ItemService;