Skip to content

Commit

Permalink
[Jenkins] merging 'release/0.9' into 'master'
Browse files Browse the repository at this point in the history
  • Loading branch information
jenkins committed May 12, 2021
2 parents 8d5f396 + 8a67b34 commit e74b485
Show file tree
Hide file tree
Showing 6 changed files with 175 additions and 7 deletions.
24 changes: 24 additions & 0 deletions .github/pull_request_template.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
# Guidelines for Pull Requests

If you haven't yet read our code review guidelines, please do so, You can find them [here](https://diging.atlassian.net/wiki/spaces/DIGING/pages/2256076801/Code+Review+Guidelines).

Please confirm the following by adding an x for each item (turn `[ ]` into `[x]`).

- [ ] I have removed all code style changes that are not necessary (e.g. changing blanks across the whole file that don’t need to be changed, adding empty lines in parts other than your own code)
- [ ] I am not making any changes to files that don’t have any effect (e.g. imports added that don’t need to be added)
- [ ] I do not have any sysout statements in my code or commented out code that isn’t needed anymore
- [ ] I am not reformatting any files in the wrong format or without cause.
- [ ] I am not changing file encoding or line endings to something else than UTF-8, LF
- [ ] My pull request does not show an insane amount of files being changed although my ticket only requires a few files being changed
- [ ] I have added Javadoc/documentation where appropriate
- [ ] I have added test cases where appropriate
- [ ] I have explained any part of my code/implementation decisions that is not be self-explanatory

## Please provide a brief description of your ticket
(you can copy the ticket if it hasn't changed)

...Put description here...

## Anything else the reviewer needs to know?

... describe here ...
2 changes: 1 addition & 1 deletion spring-social-zotero/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>edu.asu.diging</groupId>
<artifactId>spring-social-zotero</artifactId>
<version>0.8-SNAPSHOT</version>
<version>0.9</version>

<name>${project.groupId}:${project.artifactId}</name>
<url>https://github.com/diging/spring-social-zotero</url>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@

import org.springframework.social.zotero.exception.ZoteroConnectionException;

import com.fasterxml.jackson.core.JsonProcessingException;

public interface GroupsOperations {

Item[] getGroupItems(String groupId, int start, int numberOfItems, String sortBy);
Expand All @@ -24,13 +26,15 @@ ZoteroResponse<Item> getGroupItemsTop(String groupId, int start, int numberOfIte
void updateItem(String groupId, Item item, List<String> ignoreFields, List<String> validCreatorTypes)
throws ZoteroConnectionException;

ZoteroUpdateItemsStatuses batchUpdateItems(String groupId, List<Item> items, List<List<String>> ignoreFieldsList,
List<List<String>> validCreatorTypesList) throws ZoteroConnectionException, JsonProcessingException;

Long getGroupItemVersion(String groupId, String itemKey);

ItemCreationResponse createItem(String groupId, Item item, List<String> ignoreFields,
List<String> validCreatorTypes) throws ZoteroConnectionException;

void deleteItem(String groupId, String citationKey, Long citationVersion)
throws ZoteroConnectionException;
void deleteItem(String groupId, String citationKey, Long citationVersion) throws ZoteroConnectionException;

ZoteroResponse<Item> getGroupItemsByKey(String groupId, List<String> keys, boolean includeTrashed);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,7 @@ public String getMessage() {
}
public void setMessage(String message) {
this.message = message;
}

}
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
package org.springframework.social.zotero.api;

import java.util.List;

import org.springframework.social.zotero.api.ItemCreationResponse.FailedMessage;

public class ZoteroUpdateItemsStatuses {
private List<String> successItems;
private List<String> failedItems;
private List<String> unchagedItems;
private List<FailedMessage> failedMessages;

public void setSuccessItems(List<String> successItems) {
this.successItems = successItems;
}

public List<String> getSuccessItems() {
return successItems;
}

public void setFailedItems(List<String> failedItems) {
this.failedItems = failedItems;
}

public List<String> getFailedItems() {
return failedItems;
}

public void setUnchangedItems(List<String> unchagedItems) {
this.unchagedItems = unchagedItems;
}

public List<String> getUnchangedItems() {
return unchagedItems;
}

public List<FailedMessage> getFailedMessages(){
return failedMessages;
}

public void setFailedMessages(List<FailedMessage> failedMessages) {
this.failedMessages = failedMessages;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,15 @@

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.Stream;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -22,9 +27,11 @@
import org.springframework.social.zotero.api.GroupsOperations;
import org.springframework.social.zotero.api.Item;
import org.springframework.social.zotero.api.ItemCreationResponse;
import org.springframework.social.zotero.api.ItemCreationResponse.FailedMessage;
import org.springframework.social.zotero.api.ZoteroFields;
import org.springframework.social.zotero.api.ZoteroRequestHeaders;
import org.springframework.social.zotero.api.ZoteroResponse;
import org.springframework.social.zotero.api.ZoteroUpdateItemsStatuses;
import org.springframework.social.zotero.exception.ZoteroConnectionException;
import org.springframework.web.client.RestClientException;
import org.springframework.web.client.RestTemplate;
Expand All @@ -34,12 +41,15 @@
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializerProvider;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.ser.FilterProvider;
import com.fasterxml.jackson.databind.ser.PropertyWriter;
import com.fasterxml.jackson.databind.ser.impl.SimpleBeanPropertyFilter;
import com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider;

public class GroupsTemplate extends AbstractZoteroOperations implements GroupsOperations {

private static final int ZOTERO_BATCH_UPDATE_LIMIT = 50;

private final Logger logger = LoggerFactory.getLogger(getClass());

Expand Down Expand Up @@ -235,7 +245,6 @@ public Long getGroupItemVersion(String groupId, String itemKey) {
public void updateItem(String groupId, Item item, List<String> ignoreFields, List<String> validCreatorTypes)
throws ZoteroConnectionException {
String url = String.format("groups/%s/%s/%s", groupId, "items", item.getKey());

HttpHeaders headers = new HttpHeaders();
headers.set("If-Unmodified-Since-Version", item.getData().getVersion() + "");
JsonNode dataAsJson = createDataJson(item, ignoreFields, validCreatorTypes, false);
Expand All @@ -247,6 +256,60 @@ public void updateItem(String groupId, Item item, List<String> ignoreFields, Lis
throw new ZoteroConnectionException("Could not update item.", e);
}
}



/**
* This method makes a batch request call to Zotero to update items
*
* @param groupId group id of citations
* @param items items that have to be updated
* @param ignoreFieldsList fields that are not necessary while updating citations
* @param validCreatorTypesList valid creator types list
*
* @return ZoteroUpdateItemsStatuses returns items statuses
*/
@Override
public ZoteroUpdateItemsStatuses batchUpdateItems(String groupId, List<Item> items,
List<List<String>> ignoreFieldsList, List<List<String>> validCreatorTypesList)
throws ZoteroConnectionException,JsonProcessingException {
List<ItemCreationResponse> responses = new ArrayList<>();
int totalItems = items.size() - 1;
int itemsDone = 0;
List<String> itemsKeys = new ArrayList<>();

while (totalItems >= itemsDone) {
int count = 0;
List<JsonNode> dataAsJsonArray = new ArrayList<>();
for (; itemsDone <= totalItems && count < ZOTERO_BATCH_UPDATE_LIMIT; count++, itemsDone++) {
dataAsJsonArray.add(createDataJson(items.get(itemsDone), ignoreFieldsList.get(itemsDone),
validCreatorTypesList.get(itemsDone), false));
itemsKeys.add(items.get(itemsDone).getKey());
}

ObjectMapper mapper = new ObjectMapper();
ArrayNode arrayNode = mapper.createArrayNode();
arrayNode.addAll(dataAsJsonArray);
String jsonArrayString = mapper.writerWithDefaultPrettyPrinter().writeValueAsString(arrayNode);

HttpHeaders headers = new HttpHeaders();
HttpEntity<String> data = new HttpEntity<String>(jsonArrayString, headers);
String url = String.format("groups/%s/%s", groupId, "items/");

try {
ItemCreationResponse response = restTemplate
.exchange(buildUri(url, false), HttpMethod.POST, data, ItemCreationResponse.class).getBody();
responses.add(response);
} catch (RestClientException e) {
// If an exception occurs here, stop the execution. Log it and return responses
// so far. All unprocessed itemsKeys will be added to failedKeys list by
// getStatusesFromResponse() method
logger.error("Zotero connection exception occured.", e);
return getStatusesFromResponse(responses, itemsKeys);
}
}
return getStatusesFromResponse(responses, itemsKeys);
}

@Override
public ItemCreationResponse createItem(String groupId, Item item, List<String> ignoreFields,
Expand All @@ -266,7 +329,7 @@ public ItemCreationResponse createItem(String groupId, Item item, List<String> i
throw new ZoteroConnectionException("Could not create item.", e);
}
}

private JsonNode createDataJson(Item item, List<String> ignoreFields, List<String> validCreatorTypes,
boolean asArray) throws ZoteroConnectionException {
FilterProvider filters = new SimpleFilterProvider().addFilter("dataFilter", new ZoteroFieldFilter(ignoreFields))
Expand Down Expand Up @@ -350,4 +413,38 @@ public void deleteItem(String groupId, String citationKey, Long citationVersion)
}

}

private ZoteroUpdateItemsStatuses getStatusesFromResponse(List<ItemCreationResponse> responses,
List<String> itemsKeys) {
// Zotero is not sending failed items keys in response. So, we have to compute
// failed items keys on our own using success and unchanged keys

ZoteroUpdateItemsStatuses statuses = new ZoteroUpdateItemsStatuses();
List<String> successKeys = new ArrayList<>();
List<String> unchangedKeys = new ArrayList<>();
List<FailedMessage> failedMessages = new ArrayList<>();

for (ItemCreationResponse response : responses) {
successKeys.addAll(extractItemKeys(response.getSuccess(), e -> e.getValue()));
unchangedKeys.addAll(extractItemKeys(response.getUnchanged(), e -> e.getValue()));
failedMessages.addAll(
response.getFailed().entrySet().stream().map(Map.Entry::getValue).collect(Collectors.toList()));
}

statuses.setSuccessItems(successKeys);
statuses.setUnchangedItems(unchangedKeys);
statuses.setFailedMessages(failedMessages);

Set<String> successUnchangedKeys = Stream.of(successKeys, unchangedKeys).flatMap(Collection::stream)
.collect(Collectors.toSet());
List<String> failedKeys = new ArrayList<>();
failedKeys = itemsKeys.stream().filter(e -> !successUnchangedKeys.contains(e)).collect(Collectors.toList());
statuses.setFailedItems(failedKeys);

return statuses;
}

private List<String> extractItemKeys(Map<String, String> map, Function<Map.Entry<String, String>, String> keyExtractor) {
return map.entrySet().stream().map(e -> keyExtractor.apply(e)).collect(Collectors.toList());
}
}

0 comments on commit e74b485

Please sign in to comment.