[Th2-5137] Added trim-whitespace option. (#31)

th2-net · Nov 30, 2023 · 927a96f · 927a96f
1 parent f8a035b
commit 927a96f
Show file tree

Hide file tree

Showing 10 changed files with 537 additions and 426 deletions.
diff --git a/README.md b/README.md
@@ -1,4 +1,4 @@
-# Csv codec (5.1.0)
+# Csv codec (5.2.0)
 
 ## Description
 
@@ -74,9 +74,11 @@ encoding: UTF-8
 display-name: CodecCsv
 validate-length: true
 publish-header: false
+trim-whitespace: true
 ```
 
 **default-header** - the default header for this codec. It will be used if no header found in the received batch.
+  codec-csv trims all values in `default-header` and executes blank check. 
 
 **delimiter** - the delimiter to split values in received data. The default value is `,`.
 
@@ -89,6 +91,8 @@ attached to that root event. The default value for the name is `CodecCsv`.
 
 **publish-header** - set to publish decoded header. The default value is `false`.
 
+**trim-whitespace** - set to trim whitespace in header and cell. The default value is `true`.
+
 ## Full configuration example
 
 ```yaml
@@ -137,6 +141,13 @@ spec:
 
 ## Release notes
 
+### 5.2.0
+
++ Added `trim-whitespace` option.
++ Updated common:5.7.1-dev
++ Updated common-utils:2.2.2-dev
++ Updated codec:5.4.0-dev
+
 ### 5.1.0
 
 + Supports th2 transport protocol

diff --git a/build.gradle b/build.gradle
@@ -17,9 +17,9 @@ apply plugin: 'kotlin-kapt'
 
 dependencies {
     api platform("com.exactpro.th2:bom:4.5.0")
-    implementation "com.exactpro.th2:common:5.4.0-dev"
-    implementation "com.exactpro.th2:codec:5.3.0-dev"
-    implementation "com.exactpro.th2:common-utils:2.2.0-dev"
+    implementation "com.exactpro.th2:common:5.7.1-dev"
+    implementation "com.exactpro.th2:codec:5.4.0-dev"
+    implementation "com.exactpro.th2:common-utils:2.2.2-dev"
 
     implementation "org.apache.commons:commons-lang3"
     implementation "com.fasterxml.jackson.core:jackson-databind"

diff --git a/gradle.properties b/gradle.properties
@@ -1,2 +1,2 @@
-release_version=5.1.0
+release_version=5.2.0
 app_main_class=com.exactpro.th2.codec.MainKt
diff --git a/src/main/java/com/exactpro/th2/codec/csv/cfg/CsvCodecConfiguration.java b/src/main/java/com/exactpro/th2/codec/csv/cfg/CsvCodecConfiguration.java
@@ -16,14 +16,14 @@
 
 package com.exactpro.th2.codec.csv.cfg;
 
-import java.nio.charset.StandardCharsets;
-import java.util.List;
-
 import com.exactpro.th2.codec.api.IPipelineCodecSettings;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonPropertyDescription;
 import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
 
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
 public class CsvCodecConfiguration implements IPipelineCodecSettings {
     @JsonProperty("default-header")
     @JsonPropertyDescription("The default header that will be used for parsing received batch if no header found in the batch")
@@ -48,6 +48,10 @@ public class CsvCodecConfiguration implements IPipelineCodecSettings {
     @JsonPropertyDescription("Set to enable header publication")
     private boolean publishHeader = false;
 
+    @JsonProperty("trim-whitespace")
+    @JsonPropertyDescription("Set to trim whitespace in header (when default-header isn't set) and cell")
+    private boolean trimWhitespace = true;
+
     public List<String> getDefaultHeader() {
         return defaultHeader;
     }
@@ -95,4 +99,12 @@ public boolean isPublishHeader() {
     public void setPublishHeader(boolean publishHeader) {
         this.publishHeader = publishHeader;
     }
+
+    public boolean isTrimWhitespace() {
+        return trimWhitespace;
+    }
+
+    public void setTrimWhitespace(boolean trimWhitespace) {
+        this.trimWhitespace = trimWhitespace;
+    }
 }
diff --git a/src/main/kotlin/com/exactpro/th2/codec/csv/AbstractDecoder.kt b/src/main/kotlin/com/exactpro/th2/codec/csv/AbstractDecoder.kt
@@ -29,7 +29,8 @@ abstract class AbstractDecoder<ANY_MESSAGE, RAW_MESSAGE, PARSED_MESSAGE, BODY_FI
     private val csvDelimiter: Char,
     private val defaultHeader: Array<String>?,
     private val publishHeader: Boolean,
-    private val validateLength: Boolean
+    private val validateLength: Boolean,
+    private val trimWhitespace: Boolean
 ) {
     protected abstract val RAW_MESSAGE.messageMetadata: Map<String, String>
     protected abstract val RAW_MESSAGE.messageSessionAlias: String
@@ -103,7 +104,9 @@ abstract class AbstractDecoder<ANY_MESSAGE, RAW_MESSAGE, PARSED_MESSAGE, BODY_FI
                 errors.add(ErrorHolder("Empty raw at $currentIndex index (starts with 1)", rawMessage))
                 continue
             }
-            trimEachElement(strings)
+            if (trimWhitespace) {
+                trimEachElement(strings)
+            }
             if (header == null) {
                 LOGGER.debug { "Set header to: ${strings.contentToString()}" }
                 header = strings
@@ -121,7 +124,7 @@ abstract class AbstractDecoder<ANY_MESSAGE, RAW_MESSAGE, PARSED_MESSAGE, BODY_FI
                 )
                 LOGGER.error(msg)
                 LOGGER.debug { rawMessage.toString() }
-                errors.add(ErrorHolder(msg, rawMessage, strings))
+                errors.add(ErrorHolder(msg, rawMessage))
             }
 
             val headerLength = header.size
@@ -162,6 +165,7 @@ abstract class AbstractDecoder<ANY_MESSAGE, RAW_MESSAGE, PARSED_MESSAGE, BODY_FI
         try {
             ByteArrayInputStream(body).use {
                 val reader = CsvReader(it, csvDelimiter, charset)
+                reader.trimWhitespace = trimWhitespace
                 return try {
                     val result: MutableList<Array<String>> = ArrayList()
                     while (reader.readRecord()) {
@@ -179,8 +183,7 @@ abstract class AbstractDecoder<ANY_MESSAGE, RAW_MESSAGE, PARSED_MESSAGE, BODY_FI
 
     private class ErrorHolder<T>(
         val text: String,
-        val originalMessage: T,
-        val currentRow: Array<String> = emptyArray()
+        val originalMessage: T
     )
 
     private fun trimEachElement(elements: Array<String>) {

diff --git a/src/main/kotlin/com/exactpro/th2/codec/csv/CsvCodec.kt b/src/main/kotlin/com/exactpro/th2/codec/csv/CsvCodec.kt
@@ -32,8 +32,22 @@ class CsvCodec(private val config: CsvCodecConfiguration) : IPipelineCodec {
         LOGGER.info { "Default header: ${config.defaultHeader}" }
     }
 
-    private val protoDecoder = ProtoDecoder(charset, config.delimiter, defaultHeader, config.isPublishHeader, config.validateLength)
-    private val transportDecoder = TransportDecoder(charset, config.delimiter, defaultHeader, config.isPublishHeader, config.validateLength)
+    private val protoDecoder = ProtoDecoder(
+        charset,
+        config.delimiter,
+        defaultHeader,
+        config.isPublishHeader,
+        config.validateLength,
+        config.isTrimWhitespace,
+    )
+    private val transportDecoder = TransportDecoder(
+        charset,
+        config.delimiter,
+        defaultHeader,
+        config.isPublishHeader,
+        config.validateLength,
+        config.isTrimWhitespace,
+    )
 
     override fun decode(messageGroup: ProtoMessageGroup, context: IReportingContext): ProtoMessageGroup {
         val decodedMessages = protoDecoder.decode(messageGroup.messagesList)

diff --git a/src/main/kotlin/com/exactpro/th2/codec/csv/ProtoDecoder.kt b/src/main/kotlin/com/exactpro/th2/codec/csv/ProtoDecoder.kt
@@ -35,8 +35,16 @@ class ProtoDecoder(
     csvDelimiter: Char,
     defaultHeader: Array<String>?,
     publishHeader: Boolean,
-    validateLength: Boolean
-) : AbstractDecoder<AnyMessage, RawMessage, RawMessage, Value>(charset, csvDelimiter, defaultHeader, publishHeader, validateLength) {
+    validateLength: Boolean,
+    trimWhitespace: Boolean
+) : AbstractDecoder<AnyMessage, RawMessage, RawMessage, Value>(
+    charset,
+    csvDelimiter,
+    defaultHeader,
+    publishHeader,
+    validateLength,
+    trimWhitespace,
+) {
 
     override val RawMessage.messageMetadata: Map<String, String> get() = metadata.propertiesMap
     override val RawMessage.messageSessionAlias: String get() = sessionAlias ?: error("No session alias in message")

diff --git a/src/main/kotlin/com/exactpro/th2/codec/csv/TransportDecoder.kt b/src/main/kotlin/com/exactpro/th2/codec/csv/TransportDecoder.kt
@@ -28,8 +28,16 @@ class TransportDecoder(
     csvDelimiter: Char,
     defaultHeader: Array<String>?,
     publishHeader: Boolean,
-    validateLength: Boolean
-) : AbstractDecoder<Message<*>, RawMessage, ParsedMessage, Any>(charset, csvDelimiter, defaultHeader, publishHeader, validateLength) {
+    validateLength: Boolean,
+    trimWhitespace: Boolean
+) : AbstractDecoder<Message<*>, RawMessage, ParsedMessage, Any>(
+    charset,
+    csvDelimiter,
+    defaultHeader,
+    publishHeader,
+    validateLength,
+    trimWhitespace,
+) {
 
     override val RawMessage.messageMetadata: Map<String, String> get() = this.metadata
     override val RawMessage.messageSessionAlias: String get() = id.sessionAlias