Merge pull request #938 from HL7/do-20240807-fsh-2-byte-utf-8

Fix issues with UTF-8 2-byte characters
HL7 · Aug 7, 2024 · 9bfa3b4 · 9bfa3b4
2 parents c8bfb7c + ae9b857
commit 9bfa3b4
Show file tree

Hide file tree

Showing 2 changed files with 31 additions and 11 deletions.
diff --git a/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/FSHRunner.java b/org.hl7.fhir.publisher.core/src/main/java/org/hl7/fhir/igtools/publisher/FSHRunner.java
@@ -1,9 +1,7 @@
 package org.hl7.fhir.igtools.publisher;
 
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.OutputStream;
+import java.io.*;
+import java.nio.charset.StandardCharsets;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -155,17 +153,17 @@ protected String getSushiCommandString(String fshVersion, Publisher.IGBuildMode
 
     public static class MySushiHandler extends OutputStream {
 
-        private final StringBuilder buffer;
+        private final ByteArrayOutputStream buffer;
         private int errorCount = -1;
         private final Consumer<String> outputConsumer;
 
         public MySushiHandler(final Consumer<String> outputConsumer) {
-            buffer = new StringBuilder(256);
+            buffer = new ByteArrayOutputStream(256);
             this.outputConsumer = Objects.requireNonNull(outputConsumer);
         }
 
         public String getBufferString() {
-            return this.buffer.toString();
+            return this.buffer.toString(StandardCharsets.UTF_8);
         }
 
         private boolean passSushiFilter(String s) {
@@ -176,7 +174,11 @@ private boolean passSushiFilter(String s) {
 
         @Override
         public void write(int b) throws IOException {
-            this.buffer.appendCodePoint(b);
+            if (b >= -128 && b <= 127) {
+                this.buffer.write(b);
+            } else {
+                this.buffer.write('?');
+            }
             if (b == 10) { // eoln
                 final String s = this.getBufferString();
                 if (passSushiFilter(s)) {
@@ -185,7 +187,7 @@ public void write(int b) throws IOException {
                         errorCount = Integer.parseInt(s.substring(10).trim());
                     }
                 }
-                this.buffer.setLength(0);
+                this.buffer.reset();
             }
         }
 

diff --git a/org.hl7.fhir.publisher.core/src/test/java/org/hl7/fhir/igtools/publisher/FSHRunnerTest.java b/org.hl7.fhir.publisher.core/src/test/java/org/hl7/fhir/igtools/publisher/FSHRunnerTest.java
@@ -3,10 +3,12 @@
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
 import java.io.IOException;
+import java.nio.charset.StandardCharsets;
 import java.util.function.Consumer;
 
 import javax.annotation.Nonnull;
 
+import org.junit.jupiter.api.DisplayName;
 import org.junit.jupiter.api.Test;
 
 /**
@@ -21,8 +23,8 @@ void testMySushiHandler() throws IOException {
 
         final StringBuilder stringBuilder = new StringBuilder(512);
         final FSHRunner.MySushiHandler mySushiHandler = getMySushiHandler(stringBuilder);
-        for (final int codePoint : "1234".codePoints().toArray()) {
-            mySushiHandler.write(codePoint);
+        for (final int myByte : "1234".getBytes(StandardCharsets.UTF_8)) {
+            mySushiHandler.write(myByte);
         }
         assertEquals("1234", mySushiHandler.getBufferString());
         assertEquals(0, stringBuilder.length());
@@ -31,6 +33,22 @@ void testMySushiHandler() throws IOException {
         assertEquals("Sushi: 1234", stringBuilder.toString());
     }
 
+    @Test
+    @DisplayName("Test MySushiHandler with UTF-8 two-byte characters")
+    void testMySushiHandlerTwoByte() throws IOException {
+
+        final StringBuilder stringBuilder = new StringBuilder(512);
+        final FSHRunner.MySushiHandler mySushiHandler = getMySushiHandler(stringBuilder);
+        for (final int myByte : "МЗРФ".getBytes(StandardCharsets.UTF_8)) {
+            mySushiHandler.write(myByte);
+        }
+        assertEquals("МЗРФ", mySushiHandler.getBufferString());
+        assertEquals(0, stringBuilder.length());
+        mySushiHandler.write(10); // EOL
+        assertEquals("", mySushiHandler.getBufferString());
+        assertEquals("Sushi: МЗРФ", stringBuilder.toString());
+    }
+
     @Nonnull
     private static FSHRunner.MySushiHandler getMySushiHandler(StringBuilder stringBuilder) {
         final Consumer<String> stringConsumer = stringBuilder::append;