Merge pull request #53 from studio-b12/am/BodyParsing

changes to response body parsing

docs/book/src/goatfile/index.md

@@ -14,14 +14,14 @@ GET https://api.github.com/repos/{{.repo}}
 
 [Script]
 assert(response.StatusCode === 200, `Invalid response code: ${response.StatusCode}`);
-assert(response.BodyJson.language === "Go");
+assert(response.Body.language === "Go");
 
 ---
 
 GET https://api.github.com/repos/{{.repo}}/languages
 
 [Script]
-info('Languages:\n' + JSON.stringify(response.BodyJson, null, 2));
+info('Languages:\n' + JSON.stringify(response.Body, null, 2));
 assert(response.StatusCode === 200, `Invalid response code: ${response.StatusCode}`);
 ```
 

docs/book/src/goatfile/requests/options.md

@@ -77,3 +77,10 @@ Defines if a request shall be executed or not. This is useful in combination wit
 - **Default**: `0`
 
 A duration formatted as a Go [time.ParseDuration](https://pkg.go.dev/time#ParseDuration) compatible string. Execution will pause for this duration before the request is executed.
+
+### `responsetype`
+
+- **Type**: `string` 
+- **Default**: `""` 
+
+Explicit type declaration for body parsing. Implicit body parsing (json/xml) can be prevented by setting this option to `raw`.

docs/book/src/goatfile/requests/prescript.md

@@ -37,7 +37,7 @@ POST http://example.com/api/user
 
 [Script]
 assert(response.StatusCode === 201);
-var user = response.BodyJson;
+var user = response.Body;
 
 ---
 

docs/book/src/goatfile/requests/script.md

@@ -21,7 +21,7 @@
 [Script]
 ```
 assert(response.StatusCode === 200, `Response status code was ${response.StatusCode}`);
-assert(response.BodyJson.UserName === "Foo Bar");
+assert(response.Body.UserName === "Foo Bar");
 ```
 ````
 
@@ -44,12 +44,14 @@ type Response struct {
 	ProtoMinor    int
 	Header        map[string][]string
 	ContentLength int64
-	Body          string
-	BodyJson      any
+	BodyRaw       []byte
+	Body          any
 }
 ```
 
-`BodyJson` is a special field containing the response body content as a JavaScript object which will be populated if the response body can be parsed as a JSON object.
+`Body` is a special field containing the response body content as a JavaScript object which will be populated if the response body can be parsed.
+Parsers are currently implemented for `json` and `xml` and are chosen depending on the `responsetype` option or the `Content-Type` header.
+If neither are set, the raw response string gets set as `Body`. By setting the `responsetype` to `raw`, implicit body parsing can be prevented.
 
 In any script section, a number of built-in functions like `assert` can be used, which are documented [here](../../scripting/builtins.md).
 

docs/goatfile-spec.md

@@ -199,7 +199,7 @@ The script parts is getting passed builtin functions as well as the response con
 [Script]
 assert(response.StatusCode == 200);
 info(response.Body);
-var uid = response.BodyJson.Uid;
+var uid = response.Body.Uid;
 ```
 
 It is also possible to import a file as script. Simply specify the path to the file with a leading `@` to import the file by a relative path from the current Goatfile or an absolute path. The past must be specified in Unix style. Template placeholders in imported body data **will** be infused on execution!
@@ -264,7 +264,7 @@ POST {{.instance}}/api/auth/token
 [Script]
 assert(response.StatusCode == 200);
 
-var bearerToken = response.BodyJson.bearer_token;
+var bearerToken = response.Body.bearer_token;
 
 ---
 

docs/implementation.md

@@ -17,6 +17,7 @@ Name | Type | Default | Description
 `alwaysabort` | `boolean` | `false` | Forces the batch to abort when the request fails, even when the `--no-abort` flag is set.
 `condition` | `boolean` | `true` | Whether or not to execute the request.
 `delay` | `string` | `0` | A duration which is awaited before the request is executed. The duration must be formatted in compatibility to Go's [ParseDuration](https://pkg.go.dev/time#ParseDuration) function.
+`responsetype` | `string` | `""` | Explicit type declaration for body parsing. Implicit body parsing (json/xml) can be prevented by setting this option to `raw`.
 
 ### `PreScript`
 
@@ -35,8 +36,8 @@ var body = JSON.stringify({"foo": "bar"});
 
 [Script]
 assert(response.StatusCode === 200);
-assert(response.BodyJson.path === "/somepath");
-assert(response.BodyJson.body_string === '{"foo":"bar"}\n');
+assert(response.Body.path === "/somepath");
+assert(response.Body.body_string === '{"foo":"bar"}\n');
 ```
 
 ## Script Implementation
@@ -96,13 +97,13 @@ Response {
 	ProtoMinor    int
 	Header        map[string][]string
 	ContentLength int64
-	Body          string
-	BodyJson      object
+	BodyRaw       []byte
+	Body		  any
 }
 ```
-
-When the response body is JSON-parsable, `BodyJson` contains the parsed JSON as a JavaScript object. Otherwise, it will be `null`.
-
+`Body` is a special field containing the response body content as a JavaScript object which will be populated if the response body can be parsed.
+Parsers are currently implemented for `json` and `xml` and are chosen depending on the `responsetype` option or the `Content-Type` header.
+If neither are set, the raw response string gets set as `Body`. By setting the `responsetype` to `raw`, implicit body parsing can be prevented.
 
 ## Parameters
 

e2e/cases/defaults/test.goat

@@ -11,5 +11,5 @@ some body
 GET {{.instance}}/
 
 [Script]
-assert(response.BodyJson.headers["Foo"] == "bar");
-assert(response.BodyJson.body_string == "some body\n");
+assert(response.Body.headers["Foo"] == "bar");
+assert(response.Body.body_string == "some body\n");

e2e/cases/execute/c/c.goat

@@ -6,4 +6,4 @@ Hello world!
 [Script]
 assert(response.StatusCode === 200);
 println(response);
-var body = response.BodyJson.body_string.trim();
+var body = response.Body.body_string.trim();

e2e/cases/fileimport-parameterized/a/_a.goat

@@ -16,7 +16,7 @@ X-Foo: bar
 [Script]
 info(response)
 
-var body = response.BodyJson;
+var body = response.Body;
 
 assert(response.StatusCode == 200);
 assert(body.method == "POST");

e2e/cases/fileimport-parameterized/direct.goat

@@ -16,7 +16,7 @@ X-Foo: bar
 [Script]
 info(response)
 
-var body = response.BodyJson;
+var body = response.Body;
 
 assert(response.StatusCode == 200);
 assert(body.method == "POST");

e2e/cases/fileimport/a/_a.goat

@@ -16,7 +16,7 @@ X-Foo: bar
 [Script]
 info(response)
 
-var body = response.BodyJson;
+var body = response.Body;
 
 assert(response.StatusCode == 200);
 assert(body.method == "POST");

e2e/cases/fileimport/direct.goat

@@ -16,7 +16,7 @@ X-Foo: bar
 [Script]
 info(response)
 
-var body = response.BodyJson;
+var body = response.Body;
 
 assert(response.StatusCode == 200);
 assert(body.method == "POST");

e2e/cases/general/test.goat

@@ -16,7 +16,7 @@ some body
 [Script]
 info(response)
 
-var body = response.BodyJson;
+var body = response.Body;
 
 assert(response.StatusCode == 200);
 assert(body.method == "POST");

e2e/cases/prescript/test.goat

@@ -18,5 +18,5 @@ var body = JSON.stringify({"foo": "bar"});
 
 [Script]
 assert(response.StatusCode === 200);
-assert(response.BodyJson.path === "/somepath");
-assert(response.BodyJson.body_string === '{"foo":"bar"}\n');
+assert(response.Body.path === "/somepath");
+assert(response.Body.body_string === '{"foo":"bar"}\n');

e2e/issues/#43/main.goat

@@ -5,4 +5,4 @@ var path = "{{ .somePath }}"
 
 [Script]
 assert(response.StatusCode === 200);
-assert(response.BodyJson.path === "/someSubstitutedPath", `Path was ${response.BodyJson.path}`);
+assert(response.Body.path === "/someSubstitutedPath", `Path was ${response.Body.path}`);

examples/github/repo.goat

@@ -4,12 +4,12 @@ GET https://api.github.com/repos/{{.repo}}
 
 [Script]
 assert(response.StatusCode === 200, `Invalid response code: ${response.StatusCode}`);
-assert(response.BodyJson.language === "Go");
+assert(response.Body.language === "Go");
 
 ---
 
 GET https://api.github.com/repos/{{.repo}}/languages
 
 [Script]
-info('Languages:\n' + JSON.stringify(response.BodyJson, null, 2));
+info('Languages:\n' + JSON.stringify(response.Body, null, 2));
 assert(response.StatusCode === 200, `Invalid response code: ${response.StatusCode}`);

examples/restful-api/test.goat

@@ -27,7 +27,7 @@ POST https://api.restful-api.dev/objects
 [Script]
 debug(response);
 assert(response.StatusCode === 200, `Invalid response code: ${response.StatusCode}`);
-var game1Id = response.BodyJson.id;
+var game1Id = response.Body.id;
 
 ---
 
@@ -50,7 +50,7 @@ POST https://api.restful-api.dev/objects
 [Script]
 debug(response);
 assert(response.StatusCode === 200, `Invalid response code: ${response.StatusCode}`);
-var game2Id = response.BodyJson.id;
+var game2Id = response.Body.id;
 
 ---
 
@@ -85,12 +85,12 @@ id = ["{{.game1Id}}", "{{.game2Id}}"]
 debug(response);
 assert(response.StatusCode === 200, `Invalid response code: ${response.StatusCode}`);
 
-var game1 = response.BodyJson[0];
+var game1 = response.Body[0];
 assert(game1.name === "Cyberpunk 2077", "game1: Invalid name");
 assert(game1.data.developer === "CD PROJECT RED", "game1: Invalid developer");
 assert(game1.data.released === "2020-12-10T00:00:00Z", "game1: Invalid publishing date");
 
-var game2 = response.BodyJson[1];
+var game2 = response.Body[1];
 assert(game2.name === "Cult of the Lamb", "game2: Invalid name");
 assert(game2.data.developer === "Massive Monster", "game2: Invalid developer");
 assert(game2.data.tags.includes("Base Building"), "game2: Tags do not include 'Base Building'");

pkg/executor/executor.go

@@ -410,7 +410,7 @@ func (t *Executor) executeRequest(eng engine.Engine, req goatfile.Request, gf go
 		return errs.WithPrefix("http request failed:", err)
 	}
 
-	resp, err := FromHttpResponse(httpResp)
+	resp, err := FromHttpResponse(httpResp, req.Options)
 	if err != nil {
 		return errs.WithPrefix("response interpretation failed:", err)
 	}

pkg/executor/response.go

@@ -20,13 +20,13 @@ type Response struct {
 	ProtoMinor    int
 	Header        map[string][]string
 	ContentLength int64
-	Body          string
-	BodyJson      any
+	BodyRaw       []byte
+	Body          any
 }
 
 // FromHttpResponse builds a Response from the
 // given Http Response reference.
-func FromHttpResponse(resp *http.Response) (Response, error) {
+func FromHttpResponse(resp *http.Response, options map[string]any) (Response, error) {
 	var r Response
 
 	r.StatusCode = resp.StatusCode
@@ -37,20 +37,37 @@ func FromHttpResponse(resp *http.Response) (Response, error) {
 	r.Header = resp.Header
 	r.ContentLength = resp.ContentLength
 
-	d, err := io.ReadAll(resp.Body)
+	data, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return Response{},
 			errs.WithPrefix("failed reading response body:", err)
 	}
 
-	if len(d) > 0 {
-		r.Body = string(d)
+	// Try to parse body depending on 'repsponsetype' option.
+	// If 'responsetype' is not set, try to use response
+	// Content-Type header instead.
+	if len(data) > 0 {
+		r.BodyRaw = data
+		responseType, ok := options["responsetype"].(string)
+		if !ok {
+			contentTypeHeader, ok := r.Header["Content-Type"]
+			if ok {
+				responseType = contentTypeHeader[0]
+			}
+		}
+
+		// responseType 'raw' prevents body parsing
+		// if required and assigns raw bytes to 'Body'
+		if responseType == "raw" {
+			r.Body = r.BodyRaw
+			return r, nil
+		}
 
-		var bodyJson any
-		err = json.Unmarshal(d, &bodyJson)
-		if err == nil {
-			r.BodyJson = bodyJson
+		parsedBody, err := parseBody(data, responseType)
+		if err != nil {
+			return Response{}, errs.WithPrefix("failed parsing body:", err)
 		}
+		r.Body = parsedBody
 	}
 
 	return r, nil
@@ -66,14 +83,14 @@ func (t Response) String() string {
 		}
 	}
 
-	if t.BodyJson != nil {
+	if t.Body != nil && json.Valid(t.BodyRaw) {
 		enc := json.NewEncoder(&sb)
 		enc.SetIndent("", "  ")
 		// This shouldn't error because it was decoded by
 		// via json.Unmarshal before.
-		enc.Encode(t.BodyJson)
-	} else if len(t.Body) != 0 {
-		sb.WriteString(t.Body)
+		enc.Encode(t.Body)
+	} else if len(t.BodyRaw) != 0 {
+		sb.WriteString(string(t.BodyRaw))
 	}
 
 	return sb.String()

pkg/executor/util.go

@@ -1,9 +1,12 @@
 package executor
 
 import (
+	"encoding/json"
+	"encoding/xml"
 	"strings"
 
 	"github.com/studio-b12/goat/pkg/clr"
+	"github.com/studio-b12/goat/pkg/errs"
 	"github.com/zekrotja/rogu/log"
 )
 
@@ -23,3 +26,24 @@ func printSeparator(head string) {
 		head,
 		strings.Repeat("-", lenSpacerRight))
 }
+
+func parseBody(data []byte, responseType string) (any, error) {
+	if responseType == "json" || strings.Contains(responseType, "application/json") {
+		var bodyJson any
+		err := json.Unmarshal(data, &bodyJson)
+		if err != nil {
+			return nil, errs.WithPrefix("failed unmarshalling json:", err)
+		}
+		return bodyJson, nil
+	} else if responseType == "xml" || strings.Contains(responseType, "text/xml") {
+		var bodyXml any
+		err := xml.Unmarshal(data, &bodyXml)
+		if err != nil {
+			return nil, errs.WithPrefix("failed unmarshalling xml:", err)
+		}
+		return bodyXml, nil
+	} else {
+		// In the default case, return data as string
+		return string(data), nil
+	}
+}

pkg/goatfile/parser_test.go

@@ -562,7 +562,7 @@ GET https://example.com
 
 [Script]
 assert(response.StatusCode == 200, "invalid status code");
-var id = response.BodyJson.id;
+var id = response.Body.id;
 
 ---
 
@@ -574,7 +574,7 @@ var id = response.BodyJson.id;
 		assert.Nil(t, err, err)
 		assert.Equal(t,
 			StringContent(`assert(response.StatusCode == 200, "invalid status code");`+
-				"\nvar id = response.BodyJson.id;\n"),
+				"\nvar id = response.Body.id;\n"),
 			res.Tests[0].(Request).Script)
 	})
 }