Merge pull request #44 from OP-TED/release/1.10.0

Prepare release 1.10.0
OP-TED · Nov 20, 2023 · ed36806 · ed36806
2 parents 6614cea + ff8afb5
commit ed36806
Show file tree

Hide file tree

Showing 42 changed files with 20,272 additions and 6,100 deletions.
diff --git a/pom.xml b/pom.xml
@@ -4,7 +4,7 @@
 
   <groupId>eu.europa.ted.eforms</groupId>
   <artifactId>eforms-sdk-analyzer</artifactId>
-  <version>1.9.0</version>
+  <version>1.10.0</version>
   <packaging>kjar</packaging>
 
   <name>eforms-sdk-analyzer</name>
@@ -29,14 +29,14 @@
     <finalName>${project.artifactId}-${project.version}</finalName>
 
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-    <project.build.outputTimestamp>2023-08-24T14:17:24Z</project.build.outputTimestamp>
+    <project.build.outputTimestamp>2023-11-20T13:50:04Z</project.build.outputTimestamp>
 
     <java.version>11</java.version>
 
     <maven.compiler.source>${java.version}</maven.compiler.source>
     <maven.compiler.target>${java.version}</maven.compiler.target>
 
-    <version.eforms-core-java>1.0.5</version.eforms-core-java>
+    <version.eforms-core-java>1.2.0</version.eforms-core-java>
     <version.efx-toolkit>2.0.0-alpha.1</version.efx-toolkit>
 
     <!-- Version - Third-party libraries -->

diff --git a/src/main/java/eu/europa/ted/eforms/sdk/analysis/FactsLoader.java b/src/main/java/eu/europa/ted/eforms/sdk/analysis/FactsLoader.java
@@ -11,12 +11,13 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.xml.sax.SAXException;
-import eu.europa.ted.eforms.sdk.analysis.domain.Label;
+
 import eu.europa.ted.eforms.sdk.analysis.domain.SvrlReport;
 import eu.europa.ted.eforms.sdk.analysis.domain.XmlNotice;
 import eu.europa.ted.eforms.sdk.analysis.domain.codelist.Codelist;
 import eu.europa.ted.eforms.sdk.analysis.domain.field.Field;
 import eu.europa.ted.eforms.sdk.analysis.domain.field.XmlStructureNode;
+import eu.europa.ted.eforms.sdk.analysis.domain.label.Label;
 import eu.europa.ted.eforms.sdk.analysis.domain.noticetype.DocumentType;
 import eu.europa.ted.eforms.sdk.analysis.domain.noticetype.NoticeType;
 import eu.europa.ted.eforms.sdk.analysis.domain.view.index.TedefoViewTemplateIndex;
@@ -31,6 +32,7 @@
 import eu.europa.ted.eforms.sdk.analysis.fact.NoticeTypeFact;
 import eu.europa.ted.eforms.sdk.analysis.fact.NoticeTypesIndexFact;
 import eu.europa.ted.eforms.sdk.analysis.fact.SvrlReportFact;
+import eu.europa.ted.eforms.sdk.analysis.fact.TranslationsIndexFact;
 import eu.europa.ted.eforms.sdk.analysis.fact.ViewTemplateFact;
 import eu.europa.ted.eforms.sdk.analysis.fact.XmlNoticeFact;
 
@@ -98,6 +100,15 @@ public DataStore<NoticeTypesIndexFact> loadNoticeTypesIndex() throws IOException
     return datastore;
   }
 
+  public DataStore<TranslationsIndexFact> loadTranslationsIndex() throws IOException {
+    logger.debug("Creating facts datastore for translations index");
+
+    final DataStore<TranslationsIndexFact> datastore = DataSource.createStore();
+    datastore.add(new TranslationsIndexFact(sdkLoader.getTranslationsIndex()));
+
+    return datastore;
+  }
+
   public DataStore<LabelFact> loadLabels()
       throws IOException, JAXBException, SAXException, ParserConfigurationException {
     logger.debug("Creating facts datastore for labels");

diff --git a/src/main/java/eu/europa/ted/eforms/sdk/analysis/SdkLoader.java b/src/main/java/eu/europa/ted/eforms/sdk/analysis/SdkLoader.java
@@ -27,7 +27,6 @@
 import com.fasterxml.jackson.databind.SerializationFeature;
 import eu.europa.ted.eforms.sdk.SdkConstants.SdkResource;
 import eu.europa.ted.eforms.sdk.analysis.domain.EFormsTrackableEntity;
-import eu.europa.ted.eforms.sdk.analysis.domain.Label;
 import eu.europa.ted.eforms.sdk.analysis.domain.SvrlReport;
 import eu.europa.ted.eforms.sdk.analysis.domain.XmlNotice;
 import eu.europa.ted.eforms.sdk.analysis.domain.codelist.Codelist;
@@ -36,6 +35,9 @@
 import eu.europa.ted.eforms.sdk.analysis.domain.field.Field;
 import eu.europa.ted.eforms.sdk.analysis.domain.field.FieldsAndNodes;
 import eu.europa.ted.eforms.sdk.analysis.domain.field.XmlStructureNode;
+import eu.europa.ted.eforms.sdk.analysis.domain.label.Label;
+import eu.europa.ted.eforms.sdk.analysis.domain.label.LanguageFileInfo;
+import eu.europa.ted.eforms.sdk.analysis.domain.label.TranslationsIndex;
 import eu.europa.ted.eforms.sdk.analysis.domain.noticetype.NoticeSubTypeForIndex;
 import eu.europa.ted.eforms.sdk.analysis.domain.noticetype.NoticeType;
 import eu.europa.ted.eforms.sdk.analysis.domain.noticetype.NoticeTypeSdk;
@@ -57,6 +59,9 @@ public class SdkLoader {
   public static final Path EXAMPLE_NOTICES = Path.of("examples", "notices");
   public static final Path EXAMPLE_REPORTS = Path.of("examples", "reports");
 
+  // TODO: Use constant from SdkResource in ECL when it is available (TEDEFO-2707)
+  public static final Path TRANSLATIONS_JSON = Path.of("translations", "translations.json");
+
   private final Path sdkRoot;
   private final ObjectMapper objectMapper;
 
@@ -176,33 +181,33 @@ public NoticeTypeSdk getNoticeTypeSdk(String noticeId, Path sdkRoot) throws IOEx
         SdkResource.NOTICE_TYPES.getPath().toString(), MessageFormat.format("{0}.json", noticeId)));
   }
 
+  public TranslationsIndex getTranslationsIndex() throws IOException {
+    return loadJsonFile(TranslationsIndex.class,
+        Path.of(sdkRoot.toString(), TRANSLATIONS_JSON.toString()));
+  }
+
   public Set<Label> getLabels()
       throws IOException, JAXBException, SAXException, ParserConfigurationException {
     final Map<String, Label> labels = new HashMap<>();
 
     Properties translationProperties = null;
 
-    DirectoryStream.Filter<Path> filter = new DirectoryStream.Filter<Path>() {
-      public boolean accept(Path file) throws IOException {
-          return !file.getFileName().toString().matches("translations\\.(xml|json)");
-      }
-    };
-
-    try (DirectoryStream<Path> dirStream = Files.newDirectoryStream(
-        Path.of(sdkRoot.toString(), SdkResource.TRANSLATIONS.getPath().toString()), filter)) {
+    TranslationsIndex translationsIndex = getTranslationsIndex();
 
-      for (Path path : dirStream) {
-        if (!Files.isDirectory(path)) {
-          translationProperties = XmlParser.loadXmlFile(Properties.class, path);
+    for (LanguageFileInfo info : translationsIndex.getFiles()) {
+      Path path = Path.of(sdkRoot.toString(),
+          SdkResource.TRANSLATIONS.getPath().toString(),
+          info.getFilename());
+      if (!Files.isDirectory(path)) {
+        translationProperties = XmlParser.loadXmlFile(Properties.class, path);
 
-          Language language = Language.valueOf(
-              path.getFileName().toString().replaceAll("^.*?_(.*?).xml$", "$1").toUpperCase());
+        Language language = Language.valueOf(
+            path.getFileName().toString().replaceAll("^.*?_(.*?).xml$", "$1").toUpperCase());
 
-          translationProperties.getEntry().stream().forEach(e -> {
-            Label label = labels.computeIfAbsent(e.getKey(), k -> new Label(k));
-            label.addTranslation(language, e.getValue());
-          });          
-        }
+        translationProperties.getEntry().stream().forEach(e -> {
+          Label label = labels.computeIfAbsent(e.getKey(), k -> new Label(k));
+          label.addTranslation(language, e.getValue());
+        });          
       }
     }
 

diff --git a/src/main/java/eu/europa/ted/eforms/sdk/analysis/domain/field/AbstractFieldProperty.java b/src/main/java/eu/europa/ted/eforms/sdk/analysis/domain/field/AbstractFieldProperty.java
@@ -2,6 +2,7 @@
 
 import java.io.Serializable;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -64,6 +65,18 @@ public Set<String> getAllNoticeTypeIds() {
         .collect(Collectors.toSet());
   }
 
+  public Set<String> getDuplicateNoticeTypeIds() {
+    List<String> values = getConstraints()
+        .stream()
+        .map(AbstractConstraint::getNoticeTypes)
+        .flatMap(List::stream)
+        .collect(Collectors.toList());
+
+    return values.stream()
+        .filter(i -> Collections.frequency(values, i) > 1)
+        .collect(Collectors.toSet());
+  }
+
   /**
    * @return All label identifiers referenced in the constraints of this field property.
    */

diff --git a/...ted/eforms/sdk/analysis/domain/Label.java → ...orms/sdk/analysis/domain/label/Label.java b/...ted/eforms/sdk/analysis/domain/Label.java → ...orms/sdk/analysis/domain/label/Label.java
@@ -1,4 +1,4 @@
-package eu.europa.ted.eforms.sdk.analysis.domain;
+package eu.europa.ted.eforms.sdk.analysis.domain.label;
 
 import java.io.Serializable;
 import java.util.HashMap;

diff --git a/src/main/java/eu/europa/ted/eforms/sdk/analysis/domain/label/LanguageFileInfo.java b/src/main/java/eu/europa/ted/eforms/sdk/analysis/domain/label/LanguageFileInfo.java
@@ -0,0 +1,25 @@
+package eu.europa.ted.eforms.sdk.analysis.domain.label;
+
+public class LanguageFileInfo {
+
+  private String twoLetterCode;
+  private String threeLetterCode;
+  private String assetType;
+  private String filename;
+
+  public String getTwoLetterCode() {
+    return twoLetterCode;
+  }
+
+  public String getThreeLetterCode() {
+    return threeLetterCode;
+  }
+
+  public String getAssetType() {
+    return assetType;
+  }
+
+  public String getFilename() {
+    return filename;
+  }
+}
diff --git a/src/main/java/eu/europa/ted/eforms/sdk/analysis/domain/label/TranslationLanguage.java b/src/main/java/eu/europa/ted/eforms/sdk/analysis/domain/label/TranslationLanguage.java
@@ -0,0 +1,46 @@
+package eu.europa.ted.eforms.sdk.analysis.domain.label;
+
+import java.util.Objects;
+
+public class TranslationLanguage implements Comparable<TranslationLanguage> {
+  private String twoLetterCode;
+  private String threeLetterCode;
+  private String description;
+
+  public String getTwoLetterCode() {
+    return twoLetterCode;
+  }
+
+  public String getThreeLetterCode() {
+    return threeLetterCode;
+  }
+
+  public String getDescription() {
+    return description;
+  }
+
+  @Override
+  public int compareTo(TranslationLanguage o) {
+    return this.twoLetterCode.compareTo(o.getTwoLetterCode());
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(twoLetterCode);
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj) {
+      return true;
+    }
+    if (obj == null) {
+      return false;
+    }
+    if (getClass() != obj.getClass()) {
+      return false;
+    }
+    TranslationLanguage other = (TranslationLanguage) obj;
+    return Objects.equals(twoLetterCode, other.twoLetterCode);
+  }
+}
diff --git a/src/main/java/eu/europa/ted/eforms/sdk/analysis/domain/label/TranslationsIndex.java b/src/main/java/eu/europa/ted/eforms/sdk/analysis/domain/label/TranslationsIndex.java
@@ -0,0 +1,21 @@
+package eu.europa.ted.eforms.sdk.analysis.domain.label;
+
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+import com.fasterxml.jackson.annotation.JsonPropertyOrder;
+
+@JsonPropertyOrder({"files", "languages"})
+public class TranslationsIndex implements Serializable {
+
+  private final List<LanguageFileInfo> files = new ArrayList<>();
+  private final List<TranslationLanguage> languages = new ArrayList<>();
+
+  public List<LanguageFileInfo> getFiles() {
+    return files;
+  }
+
+  public List<TranslationLanguage> getLanguages() {
+    return languages;
+  }
+}
diff --git a/src/main/java/eu/europa/ted/eforms/sdk/analysis/fact/FieldFact.java b/src/main/java/eu/europa/ted/eforms/sdk/analysis/fact/FieldFact.java
@@ -15,9 +15,9 @@
 import eu.europa.ted.eforms.sdk.analysis.domain.field.FieldPrivacy;
 import eu.europa.ted.eforms.sdk.analysis.domain.field.XmlElementPosition;
 import eu.europa.ted.eforms.sdk.analysis.domain.field.XmlStructureNode;
-import eu.europa.ted.eforms.sdk.analysis.util.XPathSplitter;
 import eu.europa.ted.eforms.sdk.analysis.util.XPathUtils;
-import eu.europa.ted.eforms.sdk.analysis.util.XPathSplitter.StepInfo;
+import eu.europa.ted.eforms.xpath.XPathInfo;
+import eu.europa.ted.eforms.xpath.XPathProcessor;
 
 public class FieldFact implements SdkComponentFact<String> {
   private static final long serialVersionUID = -8325643682910825716L;
@@ -120,16 +120,22 @@ public String getXpathRelative() {
     return field.getXpathRelative();
   }
 
-  public int getXpathRelativeStepCount() {
+  public int getXpathRelativeElementCount() {
     if (stepCount == 0 && getXpathRelative() != null) {
-      stepCount = XPathSplitter.getStepElementNames(getXpathRelative()).size();
+      XPathInfo xpathInfo = XPathProcessor.parse(getXpathRelative());
+      stepCount = xpathInfo.getSteps().size();
+      if (xpathInfo.isAttribute()) {
+        // we don't want to count attributes
+        stepCount--;
+      }
     }
     return stepCount;
   }
 
   public List<String> getInvalidXpathRelativeSteps() {
-    List<String> badSteps = XPathSplitter.getStepElementNames(getXpathRelative()).stream()
-      .filter(s -> XPathUtils.isAscendingStep(s))
+    List<String> badSteps = XPathProcessor.parse(getXpathRelative()).getSteps().stream()
+      .map(s -> s.getStepText())
+      .filter(x -> XPathUtils.isAscendingStep(x))
       .collect(Collectors.toList());
 
     return badSteps;
@@ -143,8 +149,7 @@ public String getType() {
    * Return true if the last step of the relative path corresponds to an XML attribute.
    */
   public boolean isAttribute() {
-    List<StepInfo> steps = XPathSplitter.getSteps(getXpathRelative());
-    return steps.get(steps.size() - 1).isAttribute();
+    return XPathProcessor.parse(getXpathRelative()).isAttribute();
   }
 
   public String getAttributeOf() {
@@ -159,6 +164,18 @@ public List<String> getAttributes() {
     return field.getAttributes();
   }
 
+  public String getIdScheme() {
+    return field.getIdScheme();
+  }
+
+  public String getSchemeName() {
+    return field.getSchemeName();
+  }
+
+  public String getPresetValue() {
+    return field.getPresetValue();
+  }
+
   /**
    * Return a stream of the dynamic properties of the field
    */
@@ -185,6 +202,23 @@ public Set<String> getAllNoticeTypes() {
     return noticeTypes;
   }
 
+  /**
+   * Return the notices types that appear more than once in the same property.
+   */
+  public Set<String> getDuplicateNoticeTypes() {
+    Set<String> noticeTypes = new HashSet<>();
+
+    // Go over all dynamic properties and collect duplicate notice types
+    getDynamicProperties()
+        .forEach(property -> {
+          if (property != null) {
+            noticeTypes.addAll(property.getDuplicateNoticeTypeIds());
+          }
+        });
+
+    return noticeTypes;
+  }
+
   public String getPrivacyCode() {
     if (field.getPrivacy() != null) {
       return field.getPrivacy().getCode().getLiteral();

diff --git a/src/main/java/eu/europa/ted/eforms/sdk/analysis/fact/LabelFact.java b/src/main/java/eu/europa/ted/eforms/sdk/analysis/fact/LabelFact.java
@@ -1,6 +1,6 @@
 package eu.europa.ted.eforms.sdk.analysis.fact;
 
-import eu.europa.ted.eforms.sdk.analysis.domain.Label;
+import eu.europa.ted.eforms.sdk.analysis.domain.label.Label;
 
 public class LabelFact implements SdkComponentFact<String> {
   private static final long serialVersionUID = -8325643682910825716L;