Merge pull request #983 from HL7/2024-11-gg-publication-fixes

2024 11 gg publication fixes

org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/PublisherProvider.java

@@ -4,12 +4,12 @@
 import java.util.ArrayList;
 import java.util.List;
 
+import org.hl7.fhir.r5.utils.sql.Provider;
 import org.hl7.fhir.r5.context.IWorkerContext;
 import org.hl7.fhir.r5.elementmodel.Manager;
 import org.hl7.fhir.r5.elementmodel.Manager.FhirFormat;
 import org.hl7.fhir.r5.elementmodel.ValidatedFragment;
 import org.hl7.fhir.r5.model.Base;
-import org.hl7.fhir.r5.utils.sql.Provider;
 import org.hl7.fhir.utilities.Utilities;
 import org.hl7.fhir.utilities.json.model.JsonObject;
 import org.hl7.fhir.utilities.npm.NpmPackage;

org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/realm/USRealmBusinessRules.java

@@ -3,6 +3,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import java.util.Set;
 
@@ -206,7 +207,7 @@ private Resource loadResourceFromPackage(NpmPackage uscore, String filename) thr
 
   private JsonObject fetchJson(String source) throws IOException {
     try {
-      HTTPResult res = ManagedWebAccess.get(source+"?nocache=" + System.currentTimeMillis());
+      HTTPResult res = ManagedWebAccess.get(Arrays.asList("web"), source+"?nocache=" + System.currentTimeMillis());
       res.checkThrowException();
       return JsonParser.parseObject(res.getContent());
     } catch (IOException e) {

org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/utils/IGRegistryScanner.java

@@ -0,0 +1,48 @@
+package org.hl7.fhir.igtools.publisher.utils;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.jar.Attributes.Name;
+
+import org.hl7.fhir.utilities.json.model.JsonObject;
+import org.hl7.fhir.utilities.json.parser.JsonParser;
+
+public class IGRegistryScanner {
+
+  private String filename;
+
+  public IGRegistryScanner(String filename) {
+    this.filename = filename;
+  }
+
+  public static void main(String[] args) throws FileNotFoundException, IOException {
+    new IGRegistryScanner(args[0]).execute();
+
+  }
+
+  private void execute() throws FileNotFoundException, IOException {
+    JsonObject reg = JsonParser.parseObject(new FileInputStream(filename));
+    if (checkReg(reg)) {
+      JsonParser.compose(reg, new FileOutputStream(filename), true);
+    }
+  }
+
+  private boolean checkReg(JsonObject reg) {
+    boolean changed = false;
+    for (JsonObject ig : reg.forceArray("guides").asJsonObjects()) {
+      if (!ig.has("language")) {
+        String name = ig.asString("npm-name");
+        if (true || name.contains("ihe.formatcode.fhir")) {
+          changed = true;
+          ig.forceArray("language").add("en");
+        } else {
+          System.out.println("No history: "+name);
+        }
+      }
+    }
+    return changed;
+  }
+
+}

org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/utils/JekyllRepatcher.java

@@ -0,0 +1,40 @@
+package org.hl7.fhir.igtools.publisher.utils;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+import org.hl7.fhir.utilities.TextFile;
+
+public class JekyllRepatcher {
+
+  public static void main(String[] args) throws FileNotFoundException, IOException {
+    new JekyllRepatcher().process(new File(args[0]));
+  }
+
+  private void process(File folder) throws FileNotFoundException, IOException {
+    for (File f : folder.listFiles()) {
+      if (f.isDirectory()) {
+        process(f);
+      } else if (f.getName().endsWith(".html")) {
+        String src = TextFile.fileToString(f);
+        int i = src.indexOf("typeof=\"foaf:Document\">");
+        if (i > -1) {
+          String hdr = src.substring(0, i+"typeof=\"foaf:Document\">".length());
+          int b = hdr.indexOf("<title>");
+          int e = hdr.indexOf("</title>");
+          String title = hdr.substring(b+7, e);
+
+          src = src.substring(i+"typeof=\"foaf:Document\">".length());
+          i = src.indexOf("</div>");
+          src = src.substring(0, i).trim();
+
+          src = "---\nlayout: page\ntitle: "+title+"\n---\n"+src.replace("\r\n", "\n");
+          TextFile.stringToFile(src, f);
+        }
+      }
+    }
+
+  }
+
+}

org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/utils/TemplateReleaser.java

@@ -10,9 +10,11 @@
 import java.util.ArrayList;
 import java.util.Date;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
 
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
@@ -136,15 +138,14 @@ public static void main(String[] args) throws Exception {
   }
 
   private void release(String source, String dest) throws Exception {
+    System.out.println("Source: "+source);
+    System.out.println("Destination: "+dest);
     SimpleDateFormat df = new SimpleDateFormat(RSS_DATE, new Locale("en", "US"));
     checkDest(dest);
     Map<String, String> currentPublishedVersions = scanForCurrentVersions(dest);
-    System.out.println("Current Published Versions");
-    for (String s : currentPublishedVersions.keySet()) {
-      System.out.println(" * "+s+"#"+currentPublishedVersions.get(s));
-    }
     Map<String, String> currentVersions = scanForCurrentVersions(source);
-    List<VersionDecision> versionsList = analyseVersions(source, currentVersions, currentPublishedVersions);
+    summary(currentPublishedVersions, currentVersions);
+    List<VersionDecision> versionsList = analyseVersions(source, currentVersions, currentPublishedVersions, dest);
     System.out.println("Actions to take");
     for (VersionDecision vd : versionsList) {
       System.out.println(" * "+vd.summary());
@@ -203,6 +204,24 @@ private void release(String source, String dest) throws Exception {
   }
 
 
+  private void summary(Map<String, String> currentPublishedVersions, Map<String, String> currentVersions) {
+    // TODO Auto-generated method stub
+
+    Set<String> all = new HashSet<>();
+    all.addAll(currentPublishedVersions.keySet());
+    all.addAll(currentVersions.keySet());
+    System.out.println("Package Summary");
+    int l = 0;
+    for (String s : all) {
+      l = Integer.max(l, s.length());
+    }
+    for (String s: Utilities.sorted(all)) {
+      String v = currentPublishedVersions.containsKey(s) ? currentPublishedVersions.get(s) : "--";
+      String nv = currentVersions.containsKey(s) ? currentVersions.get(s) : "--";
+      System.out.println("  "+Utilities.padRight(s, ' ', l)+"  "+Utilities.padRight(v, ' ', 10)+"  "+Utilities.padRight(nv, ' ', 10));      
+    }
+  }
+
   private void buildIndexPage(Map<String, String> currentVersions, String path) throws JsonException, IOException {
     XhtmlNode tbl = new XhtmlNode(NodeType.Element, "table");
     tbl.attribute("class", "grid");
@@ -299,22 +318,24 @@ private void updatePackageList(String source, VersionDecision vd) throws FileNot
     TextFile.stringToFile(pl.toJson(), Utilities.path(source, vd.getId(), "package-list.json"));
   }
 
-  private List<VersionDecision> analyseVersions(String source, Map<String, String> newList, Map<String, String> oldList) throws Exception {
+  private List<VersionDecision> analyseVersions(String source, Map<String, String> newList, Map<String, String> oldList, String dest) throws Exception {
     List<VersionDecision> res = new ArrayList<TemplateReleaser.VersionDecision>();
     for (String s : oldList.keySet()) {
-      if (!newList.containsKey(s)) {
-        throw new Exception("Existing template "+s+" not found in release set ("+newList.keySet()+")");
-      }
-      VersionDecision vd = new VersionDecision();
-      res.add(vd);
-      vd.setId(s);
-      String v = oldList.get(s);
-      String nv = newList.get(s);
-      vd.setCurrentVersion(v);
-      vd.setType(checkVersionChangeType(v, nv));
-      vd.setExplicit(vd.getType() != VersionChangeType.NONE);
-      if (vd.isExplicit()) {
-        vd.setNewVersion(nv);
+      if (!s.equals("fhir.ca.template")) {
+        if (!newList.containsKey(s)) {
+          throw new Exception("Existing template "+s+" not found in release set ("+newList.keySet()+")");
+        }
+        VersionDecision vd = new VersionDecision();
+        res.add(vd);
+        vd.setId(s);
+        String v = oldList.get(s);
+        String nv = newList.get(s);
+        vd.setCurrentVersion(v);
+        vd.setType(checkVersionChangeType(v, nv));
+        vd.setExplicit(vd.getType() != VersionChangeType.NONE);
+        if (vd.isExplicit()) {
+          vd.setNewVersion(nv);
+        }
       }
     }
     for (String s : newList.keySet()) {
@@ -339,12 +360,27 @@ private List<VersionDecision> analyseVersions(String source, Map<String, String>
       any = any || vd.type != VersionChangeType.NONE;
     }
     if (!any) {
-      throw new Exception("Nothing found to release - Cannot Proceed");
+      System.out.println("Nothing found to release - updating History pages");
+      updateHistoryPages(dest, oldList);
+      System.out.println("Done");
+      System.exit(0);
     }
 
     return res;
   }
 
+  private void updateHistoryPages(String dest, Map<String, String> oldList) throws IOException {
+    for (String s : oldList.keySet()) {
+      String folder = Utilities.path(dest, s);
+      // create history page 
+      JsonObject jh = JsonParser.parseObjectFromFile(Utilities.path(folder, "package-list.json"));
+      String history = TextFile.fileToString(Utilities.path(dest, "history.template")).replace("\r\n", "\n");
+      history = history.replace("{{id}}", s);
+      history = history.replace("{{pl}}", JsonParser.compose(jh, false).replace('\'', '`'));
+      TextFile.stringToFile(history, Utilities.path(folder, "history.html"));
+    }
+  }
+
   private String getNewVersion(VersionChangeType t, String v) {
     switch (t) {
     case MAJOR: return VersionUtilities.incMajorVersion(v);

org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/renderers/DBBuilder.java

@@ -514,7 +514,7 @@ private void addConceptProperties(CodeSystem cs, List<ConceptDefinitionComponent
           psql.setInt(4, ((Integer) pd.getUserData("db.key")).intValue());
         }
         bindString(psql, 5, p.getCode());
-        bindString(psql, 6, p.getValue().primitiveValue());
+        bindString(psql, 6, p.getValue() == null ? p.getValue().primitiveValue() : null);
         psql.executeUpdate();    
         p.setUserData("db.key", lastCPropKey);   
       }

org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/renderers/PublicationChecker.java

@@ -210,6 +210,8 @@ private void checkPublicationRequest(List<String> messages, NpmPackage npm, Pack
         check(messages, npm.version().contains("-"), "This release is not labelled as a milestone or technical correction, so should have a patch version ("+npm.version() +")"+(isHL7(npm) ? mkError() : mkWarning()));
       }
     }
+    check(messages, !"technical-correction".equals(pr.asString("mode")), "Technical Corrections are not currently supported");
+
     if (check(messages, pr.has("status"), "No publication request status found"+mkError())) {
       if (check(messages, isValidStatus(pr.asString("status")), "Proposed status for this publication is not valid (valid values: release|trial-use|update|preview|ballot|draft|normative+trial-use|normative|informative)"+mkError())) {
         summary.add(new StringPair("status", pr.asString("status")));                        

org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/web/IGRegistryMaintainer.java

@@ -74,6 +74,7 @@ public class ImplementationGuideEntry {
     private String cibuild;
     private List<PublicationEntry> releases = new ArrayList<>();
     private List<PublicationEntry> candidates = new ArrayList<>();
+    private List<String> products = new ArrayList<>();
     private JsonObject entry;
 
     public ImplementationGuideEntry(String packageId, String canonical, String title) {
@@ -112,6 +113,10 @@ public List<PublicationEntry> getCandidates() {
     public String getCategory() {
       return entry.asString("category");
     }
+
+    public List<String> getProducts() {
+      return products;
+    }
   }
 
   private String path;
@@ -132,6 +137,7 @@ public String removeTrailingSlash(String p) {
   public ImplementationGuideEntry seeIg(String packageId, String canonical, String title, String category) {
     ImplementationGuideEntry ig = new ImplementationGuideEntry(packageId, canonical, title);
     igs.add(ig);
+    ig.getProducts().add("fhir");
     ig.entry = json.getJsonArray("guides").findByStringProp("npm-name", ig.packageId);
     if (ig.entry == null) {
       ig.entry = new JsonObject();
@@ -143,10 +149,15 @@ public ImplementationGuideEntry seeIg(String packageId, String canonical, String
       ig.entry.add("authority", getAuthority(ig.canonical));
       ig.entry.add("country", getCountry(ig.canonical));
       ig.entry.add("history", getHistoryPage(ig.canonical));
+      ig.entry.forceArray("product").add("fhir");
       JsonArray a = new JsonArray();
       ig.entry.add("language", a);
       a.add(new JsonString("en"));
-    } 
+    } else {
+      for (String s : ig.entry.forceArray("product").asStrings()) {
+        ig.products.add(s);
+      }
+    }
     if (!ig.entry.has("category") && !Utilities.noString(category)) {
       ig.entry.add("category", category);      
     }
@@ -186,6 +197,10 @@ public void finish() throws FileNotFoundException, IOException {
         if (ig.entry.has("editions")) {
           ig.entry.remove("editions");
         }
+        ig.entry.remove("product");
+        for (String s : ig.products) {
+          ig.entry.forceArray("product").add(s);
+        }
 
         JsonArray a = new JsonArray();
         ig.entry.add("editions", a);

org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/web/WebSourceProvider.java

@@ -7,6 +7,7 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.time.Duration;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
@@ -84,7 +85,7 @@ public void needFile(String path) throws IOException {
       String url = Utilities.pathURL(source, path)+"?nocache=" + System.currentTimeMillis();
       System.out.println("Fetch "+ url);
       long t = System.currentTimeMillis();
-      HTTPResult res = ManagedWebAccess.get(url);
+      HTTPResult res = ManagedWebAccess.get(Arrays.asList("web"), url);
       res.checkThrowException();
       TextFile.bytesToFile(res.getContent(), df);
       System.out.println("  ... done ("+stats(res.getContent().length, t)+")");
@@ -111,7 +112,7 @@ public void needFolder(String path, boolean clear) throws IOException {
       String url = Utilities.pathURL(source, path, "_ig-pub-archive.zip")+"?nocache=" + System.currentTimeMillis();
       System.out.println("Fetch "+ url);
       long t = System.currentTimeMillis();
-      HTTPResult res = ManagedWebAccess.get(url);
+      HTTPResult res = ManagedWebAccess.get(Arrays.asList("web"), url);
       res.checkThrowException();
       folderSources.put(path, res.getContent());
       Utilities.unzip(new ByteArrayInputStream(res.getContent()), df.getAbsolutePath());
@@ -176,7 +177,7 @@ public void needOptionalFile(String path) throws IOException {
       String url = Utilities.pathURL(source, path)+"?nocache=" + System.currentTimeMillis();
       System.out.println("Fetch "+ url);
       long t = System.currentTimeMillis();
-      HTTPResult res = ManagedWebAccess.get(url);
+      HTTPResult res = ManagedWebAccess.get(Arrays.asList("web"), url);
       if (res.getCode() < 300 && res.getContent().length > 0) {
         TextFile.bytesToFile(res.getContent(), df);
       }

test-statistics.csv

@@ -35,3 +35,4 @@ Version,example.fhir.uv.myig,fhir.base.template.ig,hl7.base.template.ig,hl7.cda.
 1.6.30,70326,887,99,108146,319183,927,1601631,856561,635813,33073,95073,556207,585633,58332,256680
 1.6.31,74408,934,88,116831,302345,1085,1563643,925565,613025,42689,151744,864504,634988,81602,234197
 1.7.0,67385,947,87,116042,276551,773,1291011,711790,615710,26255,93022,569245,570301,57877,215056
+1.7.2,85673,1922,93,118751,265472,1380,1681712,1072974,742736,33289,122122,753126,701645,79859,280102