Skip to content

Commit

Permalink
Merge pull request #14 from statisticsnorway/feature/csv-fields
Browse files Browse the repository at this point in the history
Feature/csv fields
  • Loading branch information
alina-lapina authored Nov 11, 2020
2 parents f603e63 + 158b44a commit c63bce9
Show file tree
Hide file tree
Showing 18 changed files with 271 additions and 40 deletions.
2 changes: 1 addition & 1 deletion klass-api/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
<parent>
<groupId>no.ssb.klass</groupId>
<artifactId>klass-root</artifactId>
<version>2.0.18</version>
<version>2.1.0</version>
<!-- <relativePath>../</relativePath>-->
</parent>

Expand Down
19 changes: 16 additions & 3 deletions klass-api/src/main/asciidoc/api-guide.adoc
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,7 @@ Used to search codes from a classification variant. A range is specified when re
and the response will for each code indicate its valid range (validFrom/validTo).
The format and character set used, can be specified in the http header (character set is only available for csv).
For more information about the range see <<_range, range>>. +
To get a snapshot of codes valid at a specified date, use <<_variantaAt, variantAt>>.
To get a snapshot of codes valid at a specified date, use <<_variantat, variantAt>>.
See also <<_variants_by_id, variants>>

===== Supported formats
Expand Down Expand Up @@ -510,15 +510,28 @@ include::{snippets}/select-codes-example-csv/http-response.adoc[]


=== csvSeparator
`csvSeparator` is used to specify separator to be used for csv format. Default is `,`
`csvSeparator` is used to specify a separator symbol to be used for csv format. Default is `,`

===== Example request for csvSeparator=;
include::{snippets}/csv-separator-example/curl-request.adoc[]
===== Example response
include::{snippets}/csv-separator-example/http-response.adoc[]


=== csvFields
`csvFields` is an optional parameter that allows a user to filter columns and specify columns order in the Csv output.
It is applicable for `/codes`, `/codesAt`, `/variantAt`, `/correspondsAt` services.

The field names are case-sensitive. Misspelled field names will cause a 400 BAD REQUEST error.

===== Example request for csvFields=name,code with /codesAt;
include::{snippets}/csv-fields-codes-at-example/curl-request.adoc[]
===== Example response
include::{snippets}/csv-fields-codes-at-example/http-response.adoc[]


=== language
`language` is used to specify which language data shall be presented in. Default if none is selected is nb (Norwegian Bokmål).
`language` is used to specify a language data shall be presented in. If none selected, the default is "nb" (Norwegian Bokmål).

Supported languages
|===
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,11 +6,11 @@
import java.net.MalformedURLException;
import java.net.URL;
import java.time.LocalDate;
import java.util.Arrays;
import java.util.Date;
import java.util.List;

import javax.transaction.Transactional;

import no.ssb.klass.api.controllers.validators.CsvFieldsValidator;
import no.ssb.klass.core.util.AlphaNumericalComparator;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -90,15 +90,18 @@ public class ClassificationController {
private final SubscriberService subscriberService;
private final SearchService searchService;
private final StatisticsService statisticsService;
private final CsvFieldsValidator csvFieldsValidator;

@Autowired
public ClassificationController(ClassificationService classificationService,
SubscriberService subscriberService,
SearchService searchService, StatisticsService statisticsService) {
SearchService searchService, StatisticsService statisticsService,
CsvFieldsValidator csvFieldsValidator) {
this.classificationService = classificationService;
this.subscriberService = subscriberService;
this.searchService = searchService;
this.statisticsService = statisticsService;
this.csvFieldsValidator = csvFieldsValidator;
}

@ExceptionHandler
Expand All @@ -121,6 +124,12 @@ public String argumentTypeMismatchExceptionHandler(MethodArgumentTypeMismatchExc
return exception.getMessage();
}

@ExceptionHandler
@ResponseStatus(HttpStatus.BAD_REQUEST)
public String argumentTypeMismatchExceptionHandler(IllegalArgumentException exception) {
return exception.getMessage();
}

@ExceptionHandler
@ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR)
public String serverErrorExceptionHandler(Exception exception) {
Expand Down Expand Up @@ -200,7 +209,7 @@ public KlassPagedResources<SearchResultResource> search(
// @formatter:off
@RequestParam(value = "query") String query,
@RequestParam(value = "ssbSection", required = false) String ssbSection,
@RequestParam(value = "includeCodelists", defaultValue = "false") boolean includeCodelists,
@RequestParam(value = "includeCodelists", defaultValue = "false") boolean includeCodelists,
Pageable pageable, PagedResourcesAssembler<SolrSearchResult> assembler) {
// @formatter:on
Link self = new Link(getCurrentRequest(), Link.REL_SELF);
Expand Down Expand Up @@ -258,31 +267,48 @@ public CodeList codes(@PathVariable Long id,
@RequestParam(value = "from") @DateTimeFormat(pattern = RestConstants.DATE_FORMAT) LocalDate from,
@RequestParam(value = "to", required = false) @DateTimeFormat(pattern = RestConstants.DATE_FORMAT) LocalDate to,
@RequestParam(value = "csvSeparator", defaultValue = ",") String csvSeparator,
@RequestParam(value = "csvFields", defaultValue = "") String csvFields,
@RequestParam(value = "selectLevel", required = false) String selectLevel,
@RequestParam(value = "selectCodes", required = false) String selectCodes,
@RequestParam(value = "presentationNamePattern", required = false) String presentationNamePattern,
@RequestParam(value = "language", defaultValue = "nb") Language language,
@RequestParam(value = "includeFuture", defaultValue = "false") Boolean includeFuture
) {
// @formatter:on
return codesInternal(id, new DateRangeHolder(from, to), csvSeparator, selectLevel, selectCodes,
CodeList codeList = codesInternal(id, new DateRangeHolder(from, to), csvSeparator, selectLevel, selectCodes,
presentationNamePattern, language, includeFuture);

if (!csvFields.isEmpty()) {
List<String> csvFieldsList = getCsvFieldsList(csvFields);
csvFieldsValidator.validateFieldsCodeItem( csvFieldsList);
codeList.setCsvFields(csvFieldsList);
}
return codeList;
}

@RequestMapping(value = "/classifications/{id}/codesAt", method = RequestMethod.GET)
public CodeList codesAt(@PathVariable Long id,
// @formatter:off
@RequestParam(value = "date") @DateTimeFormat(pattern = RestConstants.DATE_FORMAT) LocalDate date,
@RequestParam(value = "csvSeparator", defaultValue = ",") String csvSeparator,
@RequestParam(value = "csvFields", defaultValue = "") String csvFields,
@RequestParam(value = "selectLevel", required = false) String selectLevel,
@RequestParam(value = "selectCodes", required = false) String selectCodes,
@RequestParam(value = "presentationNamePattern", required = false) String presentationNamePattern,
@RequestParam(value = "language", defaultValue = "nb") Language language,
@RequestParam(value = "includeFuture", defaultValue = "false") Boolean includeFuture
) {
// @formatter:on
return codesInternal(id, new DateRangeHolder(date), csvSeparator, selectLevel, selectCodes,

CodeList codeList = codesInternal(id, new DateRangeHolder(date), csvSeparator, selectLevel, selectCodes,
presentationNamePattern, language, includeFuture);

if (!csvFields.isEmpty()) {
List<String> csvFieldsList = getCsvFieldsList(csvFields);
csvFieldsValidator.validateFieldsCodeItem(csvFieldsList);
codeList.setCsvFields(csvFieldsList);
}
return codeList;
}

private CodeList codesInternal(Long id, DateRangeHolder dateRangeHolder, String csvSeparator, String selectLevel,
Expand All @@ -304,6 +330,7 @@ public CodeChangeList changes(@PathVariable Long id,
@RequestParam(value = "from") @DateTimeFormat(pattern = RestConstants.DATE_FORMAT) LocalDate from,
@RequestParam(value = "to", required = false) @DateTimeFormat(pattern = RestConstants.DATE_FORMAT) LocalDate to,
@RequestParam(value = "csvSeparator", defaultValue = ",") String csvSeparator,
@RequestParam(value = "csvFields", defaultValue = "") String csvFields,
@RequestParam(value = "language", defaultValue = "nb") Language language,
@RequestParam(value = "includeFuture", defaultValue = "false") Boolean includeFuture) {
// @formatter:on
Expand All @@ -315,6 +342,13 @@ public CodeChangeList changes(@PathVariable Long id,
for (CorrespondenceTable changeTable : changeTables) {
codeChanges = codeChanges.merge(codeChanges.convert(changeTable, language));
}

if (!csvFields.isEmpty()) {
List<String> csvFieldsList = getCsvFieldsList(csvFields);
csvFieldsValidator.validateFieldsChangeItemSchema(csvFieldsList);
codeChanges.setCsvFields(csvFieldsList);
}

return codeChanges;
}

Expand All @@ -325,15 +359,25 @@ public CodeList variant(@PathVariable Long id,
@RequestParam(value = "from") @DateTimeFormat(pattern = RestConstants.DATE_FORMAT) LocalDate from,
@RequestParam(value = "to", required = false) @DateTimeFormat(pattern = RestConstants.DATE_FORMAT) LocalDate to,
@RequestParam(value = "csvSeparator", defaultValue = ",") String csvSeparator,
@RequestParam(value = "csvFields", defaultValue = "") String csvFields,
@RequestParam(value = "level", required = false) String selectLevel,
@RequestParam(value = "selectCodes", required = false) String selectCodes,
@RequestParam(value = "presentationNamePattern", required = false) String presentationNamePattern,
@RequestParam(value = "language", defaultValue = "nb") Language language,
@RequestParam(value = "includeFuture", defaultValue = "false") Boolean includeFuture
) {
// @formatter:on
return variantInternal(id, variantName, new DateRangeHolder(from, to), csvSeparator, selectLevel, selectCodes,
CodeList codeList = variantInternal(id, variantName, new DateRangeHolder(from, to), csvSeparator, selectLevel, selectCodes,
presentationNamePattern, language, includeFuture);

if (!csvFields.isEmpty()) {
List<String> csvFieldsList = getCsvFieldsList(csvFields);
csvFieldsValidator.validateFieldsCodeItem(csvFieldsList);
codeList.setCsvFields(csvFieldsList);
}

return codeList;

}

@RequestMapping(value = "/classifications/{id}/variantAt", method = RequestMethod.GET)
Expand All @@ -342,17 +386,28 @@ public CodeList variantAt(@PathVariable Long id,
@RequestParam(value = "variantName") String variantName,
@RequestParam(value = "date", required = false) @DateTimeFormat(pattern = RestConstants.DATE_FORMAT) LocalDate date,
@RequestParam(value = "csvSeparator", defaultValue = ",") String csvSeparator,
@RequestParam(value = "csvFields", defaultValue = "") String csvFields,
@RequestParam(value = "level", required = false) String selectLevel,
@RequestParam(value = "selectCodes", required = false) String selectCodes,
@RequestParam(value = "presentationNamePattern", required = false) String presentationNamePattern,
@RequestParam(value = "language", defaultValue = "nb") Language language,
@RequestParam(value = "includeFuture", defaultValue = "false") Boolean includeFuture
) {
// @formatter:on
return variantInternal(id, variantName, new DateRangeHolder(date), csvSeparator, selectLevel, selectCodes,
CodeList codeList = variantInternal(id, variantName, new DateRangeHolder(date), csvSeparator, selectLevel, selectCodes,
presentationNamePattern, language, includeFuture);

if (!csvFields.isEmpty()) {
List<String> csvFieldsList = getCsvFieldsList(csvFields);
csvFieldsValidator.validateFieldsCodeItem(csvFieldsList);
codeList.setCsvFields(csvFieldsList);
}

return codeList;
}



private CodeList variantInternal(Long id, String variantName, DateRangeHolder dateRangeHolder, String csvSeparator,
String selectLevel, String selectCodes, String presentationNamePattern, Language language, Boolean includeFuture) {
List<CodeDto> codes = classificationService.findVariantClassificationCodes(id, variantName, language,
Expand All @@ -369,10 +424,19 @@ public CorrespondenceItemList corresponds(@PathVariable Long id,
@RequestParam(value = "from") @DateTimeFormat(pattern = RestConstants.DATE_FORMAT) LocalDate from,
@RequestParam(value = "to", required = false) @DateTimeFormat(pattern = RestConstants.DATE_FORMAT) LocalDate to,
@RequestParam(value = "csvSeparator", defaultValue = ",") String csvSeparator,
@RequestParam(value = "csvFields", defaultValue = "") String csvFields,
@RequestParam(value = "language", defaultValue = "nb") Language language,
@RequestParam(value = "includeFuture", defaultValue = "false") Boolean includeFuture) {
// @formatter:on
return correspondsInternal(id, targetClassificationId, new DateRangeHolder(from, to), csvSeparator, language, includeFuture);
CorrespondenceItemList correspondenceList = correspondsInternal(id, targetClassificationId, new DateRangeHolder(from, to), csvSeparator, language, includeFuture);

if (!csvFields.isEmpty()) {
List<String> csvFieldsList = getCsvFieldsList(csvFields);
csvFieldsValidator.validateFieldsCorrespondenceItem(csvFieldsList);
correspondenceList.setCsvFields(csvFieldsList);
}

return correspondenceList;
}

@RequestMapping(value = "/classifications/{id}/correspondsAt", method = RequestMethod.GET)
Expand All @@ -381,10 +445,19 @@ public CorrespondenceItemList correspondsAt(@PathVariable Long id,
@RequestParam(value = "targetClassificationId") Long targetClassificationId,
@RequestParam(value = "date", required = false) @DateTimeFormat(pattern = RestConstants.DATE_FORMAT) LocalDate date,
@RequestParam(value = "csvSeparator", defaultValue = ",") String csvSeparator,
@RequestParam(value = "csvFields", defaultValue = "") String csvFields,
@RequestParam(value = "language", defaultValue = "nb") Language language,
@RequestParam(value = "includeFuture", defaultValue = "false") Boolean includeFuture) {
// @formatter:on
return correspondsInternal(id, targetClassificationId, new DateRangeHolder(date), csvSeparator, language, includeFuture);
CorrespondenceItemList correspondenceList = correspondsInternal(id, targetClassificationId, new DateRangeHolder(date), csvSeparator, language, includeFuture);

if (!csvFields.isEmpty()) {
List<String> csvFieldsList = getCsvFieldsList(csvFields);
csvFieldsValidator.validateFieldsCorrespondenceItem(csvFieldsList);
correspondenceList.setCsvFields(csvFieldsList);
}

return correspondenceList;
}

private CorrespondenceItemList correspondsInternal(Long id, Long targetClassificationId,
Expand Down Expand Up @@ -479,6 +552,11 @@ private String extractSsbSection(String ssbSection) {
return Strings.isNullOrEmpty(ssbSection) ? null : ssbSection;
}

private List<String> getCsvFieldsList(String csvFields) {
return Arrays.asList(csvFields.split(","));
}


@InitBinder
public void initBinder(WebDataBinder binder) {
binder.registerCustomEditor(Language.class, new CaseInsensitiveConverter<>(Language.class));
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
package no.ssb.klass.api.controllers.validators;

import com.fasterxml.jackson.dataformat.csv.CsvMapper;
import com.fasterxml.jackson.dataformat.csv.CsvSchema;
import no.ssb.klass.api.dto.CodeChangeItem;
import no.ssb.klass.api.dto.CodeItem;
import no.ssb.klass.api.dto.CorrespondenceItem;
import org.springframework.stereotype.Component;

import java.util.List;

import static java.util.stream.Collectors.toList;

@Component
public class CsvFieldsValidator {

private final CsvSchema correspondenceItemSchema = new CsvMapper().schemaFor(CorrespondenceItem.RangedCorrespondenceItem.class);
private final CsvSchema codeItemSchema = new CsvMapper().schemaFor(CodeItem.class);
private final CsvSchema codeChangeItemSchema = new CsvMapper().schemaFor(CodeChangeItem.class);

public void validateFieldsCorrespondenceItem(List<String> csvFields) {
List<String> fieldsNotFound = csvFields.stream().filter(s -> correspondenceItemSchema.column(s) == null).collect(toList());
if(!fieldsNotFound.isEmpty()) {
throw new IllegalArgumentException("CorrespondenceList does not contain the following field(s): " + String.join(",", fieldsNotFound));
}
}
public void validateFieldsCodeItem(List<String> csvFields) {
List<String> fieldsNotFound = csvFields.stream().filter(s -> codeItemSchema.column(s) == null).collect(toList());
if(!fieldsNotFound.isEmpty()) {
throw new IllegalArgumentException("CorrespondenceList does not contain the following field(s): " + String.join(",", fieldsNotFound));
}
}
public void validateFieldsChangeItemSchema(List<String> csvFields) {
List<String> fieldsNotFound = csvFields.stream().filter(s -> codeChangeItemSchema.column(s) == null).collect(toList());
if(!fieldsNotFound.isEmpty()) {
throw new IllegalArgumentException("CorrespondenceList does not contain the following field(s): " + String.join(",", fieldsNotFound));
}
}

}
Loading

0 comments on commit c63bce9

Please sign in to comment.