Improve memory footprint

convert/encoding.go

@@ -1,8 +1,9 @@
 package convert
 
-// TODO: []byte -> io.ReadCloser / io.Writer
+import "io"
+
 type Encoding interface {
 	String() string
-	Encode(input interface{}) ([]byte, error)
-	Decode(input []byte) (interface{}, error)
+	Encode(w io.Writer, in interface{}) error
+	Decode(r io.Reader) (interface{}, error)
 }

convert/hcl.go

@@ -2,6 +2,8 @@ package convert
 
 import (
 	"bytes"
+	"io"
+	"io/ioutil"
 
 	"github.com/hashicorp/hcl"
 	"github.com/hashicorp/hcl/hcl/printer"
@@ -14,23 +16,23 @@ func (HCL) String() string {
 	return "HCL"
 }
 
-func (HCL) Encode(input interface{}) ([]byte, error) {
-	json, err := JSON{}.Encode(input)
+func (HCL) Encode(w io.Writer, in interface{}) error {
+	j := &bytes.Buffer{}
+	if err := (JSON{}).Encode(j, in); err != nil {
+		return err
+	}
+	ast, err := hcljson.Parse(j.Bytes())
 	if err != nil {
-		return nil, err
+		return err
 	}
+	return printer.Fprint(w, ast)
+}
 
-	ast, err := hcljson.Parse(json)
+func (HCL) Decode(r io.Reader) (interface{}, error) {
+	in, err := ioutil.ReadAll(r)
 	if err != nil {
 		return nil, err
 	}
-
-	output := &bytes.Buffer{}
-	err = printer.Fprint(output, ast)
-	return output.Bytes(), err
-}
-
-func (HCL) Decode(input []byte) (interface{}, error) {
 	var data interface{}
-	return data, hcl.Unmarshal(input, &data)
+	return data, hcl.Unmarshal(in, &data)
 }

convert/json.go

@@ -1,8 +1,8 @@
 package convert
 
 import (
-	"bytes"
 	"encoding/json"
+	"io"
 )
 
 type JSON struct {
@@ -13,15 +13,13 @@ func (JSON) String() string {
 	return "JSON"
 }
 
-func (j JSON) Encode(input interface{}) ([]byte, error) {
-	output := &bytes.Buffer{}
-	encoder := json.NewEncoder(output)
+func (j JSON) Encode(w io.Writer, in interface{}) error {
+	encoder := json.NewEncoder(w)
 	encoder.SetEscapeHTML(j.EscapeHTML)
-	err := encoder.Encode(input)
-	return output.Bytes(), err
+	return encoder.Encode(in)
 }
 
-func (JSON) Decode(input []byte) (interface{}, error) {
+func (JSON) Decode(r io.Reader) (interface{}, error) {
 	var data interface{}
-	return data, json.Unmarshal(input, &data)
+	return data, json.NewDecoder(r).Decode(&data)
 }

convert/toml.go

@@ -1,7 +1,7 @@
 package convert
 
 import (
-	"bytes"
+	"io"
 
 	"github.com/BurntSushi/toml"
 )
@@ -12,13 +12,12 @@ func (TOML) String() string {
 	return "TOML"
 }
 
-func (TOML) Encode(input interface{}) ([]byte, error) {
-	output := &bytes.Buffer{}
-	err := toml.NewEncoder(output).Encode(input)
-	return output.Bytes(), err
+func (TOML) Encode(w io.Writer, in interface{}) error {
+	return toml.NewEncoder(w).Encode(in)
 }
 
-func (TOML) Decode(input []byte) (interface{}, error) {
+func (TOML) Decode(r io.Reader) (interface{}, error) {
 	var data interface{}
-	return data, toml.Unmarshal(input, &data)
+	_, err := toml.DecodeReader(r, &data)
+	return data, err
 }

convert/yaml.go

@@ -2,6 +2,7 @@ package convert
 
 import (
 	"encoding/json"
+	"io"
 	"math"
 
 	goyaml "gopkg.in/yaml.v2"
@@ -19,22 +20,24 @@ func (YAML) String() string {
 	return "YAML"
 }
 
-func (y YAML) Encode(input interface{}) ([]byte, error) {
-	encoder := &yaml.Encoder{}
+func (y YAML) Encode(w io.Writer, in interface{}) error {
+	encoder := &yaml.Encoder{
+		EncodeYAML: goyaml.NewEncoder(w).Encode,
+	}
 	if y.FloatStrings {
 		encoder.NaN = "NaN"
 		encoder.PosInf = "Infinity"
 		encoder.NegInf = "-Infinity"
 	}
-	encoder.Marshal = goyaml.Marshal
 	if y.JSONKeys {
 		encoder.KeyUnmarshal = json.Unmarshal
 	}
-	return encoder.YAML(input)
+	return encoder.YAML(in)
 }
 
-func (y YAML) Decode(input []byte) (interface{}, error) {
+func (y YAML) Decode(r io.Reader) (interface{}, error) {
 	decoder := &yaml.Decoder{
+		DecodeYAML: goyaml.NewDecoder(r).Decode,
 		KeyMarshal: (&yaml.JSON{EscapeHTML: y.EscapeHTML}).Marshal,
 
 		NaN:    (*float64)(nil),
@@ -47,7 +50,5 @@ func (y YAML) Decode(input []byte) (interface{}, error) {
 		decoder.PosInf = "Infinity"
 		decoder.NegInf = "-Infinity"
 	}
-
-	decoder.Unmarshal = goyaml.Unmarshal
-	return decoder.JSON(input)
+	return decoder.JSON()
 }

main.go

@@ -1,10 +1,8 @@
 package main
 
 import (
-	"bytes"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 )
 
@@ -52,27 +50,14 @@ func Run(stdin io.Reader, stdout, stderr io.Writer, osArgs []string) (code int)
 		return 0
 	}
 
-	input, err := ioutil.ReadAll(stdin)
-	if err != nil {
-		fmt.Fprintf(stderr, "Error: %s\n", err)
-		return 1
-	}
-
-	if len(bytes.TrimSpace(input)) == 0 {
-		return 0
-	}
-
-	// TODO: if from == to, don't do yaml decode/encode to avoid stringifying the keys
-	rep, err := config.From.Decode(input)
+	rep, err := config.From.Decode(stdin)
 	if err != nil {
 		fmt.Fprintf(stderr, "Error parsing %s: %s\n", config.From, err)
 		return 1
 	}
-	output, err := config.To.Encode(rep)
-	if err != nil {
+	if err := config.To.Encode(stdout, rep); err != nil {
 		fmt.Fprintf(stderr, "Error writing %s: %s\n", config.To, err)
 		return 1
 	}
-	fmt.Fprintf(stdout, "%s", output)
 	return 0
 }

main_test.go

@@ -31,7 +31,7 @@ func TestRunWhenHelpFlagIsProvided(t *testing.T) {
 func TestRunWhenStdinIsInvalid(t *testing.T) {
 	stdout, stderr := &bytes.Buffer{}, &bytes.Buffer{}
 	assertEq(t, main.Run(&errReader{}, stdout, stderr, []string{""}), 1)
-	assertEq(t, strings.Contains(stderr.String(), "Error: some reader error"), true)
+	assertEq(t, strings.Contains(stderr.String(), "some reader error"), true)
 	assertEq(t, stdout.Len(), 0)
 }
 

yaml/common_test.go

@@ -6,20 +6,18 @@ import (
 )
 
 type mockYAML struct {
-	data  []byte
 	value interface{}
 	err   error
 }
 
-func (m *mockYAML) unmarshal(data []byte, v interface{}) error {
-	m.data = data
+func (m *mockYAML) decode(v interface{}) error {
 	*v.(*interface{}) = m.value
 	return m.err
 }
 
-func (m *mockYAML) marshal(v interface{}) ([]byte, error) {
+func (m *mockYAML) encode(v interface{}) error {
 	m.value = v
-	return m.data, m.err
+	return m.err
 }
 
 func assertEq(t *testing.T, a, b interface{}) {

yaml/decoder.go

@@ -7,7 +7,7 @@ import (
 )
 
 type Decoder struct {
-	Unmarshal  func([]byte, interface{}) error
+	DecodeYAML func(interface{}) error
 	KeyMarshal func(interface{}) ([]byte, error)
 
 	// If not set, input YAML must not contain these.
@@ -16,10 +16,10 @@ type Decoder struct {
 }
 
 // JSON decodes YAML into an object that marshals cleanly into JSON.
-func (d *Decoder) JSON(yaml []byte) (json interface{}, err error) {
+func (d *Decoder) JSON() (json interface{}, err error) {
 	defer catchFailure(&err)
 	var data interface{}
-	if err := d.Unmarshal(yaml, &data); err != nil {
+	if err := d.DecodeYAML(&data); err != nil {
 		return nil, err
 	}
 	return d.jsonify(data), nil

yaml/decoder_test.go

@@ -12,43 +12,42 @@ import (
 func TestDecoder(t *testing.T) {
 	mock := &mockYAML{value: yamlFixture}
 	decoder := &yaml.Decoder{
-		Unmarshal:  mock.unmarshal,
+		DecodeYAML: mock.decode,
 		KeyMarshal: json.Marshal,
 		NaN:        F{"NaN"},
 		PosInf:     F{"Infinity"},
 		NegInf:     F{"-Infinity"},
 	}
-	json, err := decoder.JSON([]byte("some YAML"))
+	json, err := decoder.JSON()
 	assertEq(t, err, nil)
 	assertEq(t, json, jsonFixture)
-	assertEq(t, mock.data, []byte("some YAML"))
 }
 
 func TestDecoderWhenYAMLIsInvalid(t *testing.T) {
 	mock := &mockYAML{err: errors.New("some error")}
-	decoder := &yaml.Decoder{Unmarshal: mock.unmarshal}
-	_, err := decoder.JSON(nil)
+	decoder := &yaml.Decoder{DecodeYAML: mock.decode}
+	_, err := decoder.JSON()
 	assertEq(t, err.Error(), "some error")
 }
 
 func TestDecoderWhenYAMLHasInvalidTypes(t *testing.T) {
 	mock := &mockYAML{}
-	decoder := &yaml.Decoder{Unmarshal: mock.unmarshal}
+	decoder := &yaml.Decoder{DecodeYAML: mock.decode}
 
 	mock.value = map[int]int{}
-	_, err := decoder.JSON(nil)
+	_, err := decoder.JSON()
 	assertEq(t, err.Error(), "unexpected type: map[int]int{}")
 
 	mock.value = [0]int{}
-	_, err = decoder.JSON(nil)
+	_, err = decoder.JSON()
 	assertEq(t, err.Error(), "unexpected type: [0]int{}")
 
 	mock.value = []int{}
-	_, err = decoder.JSON(nil)
+	_, err = decoder.JSON()
 	assertEq(t, err.Error(), "unexpected type: []int{}")
 
 	mock.value = float32(0)
-	_, err = decoder.JSON(nil)
+	_, err = decoder.JSON()
 	assertEq(t, err.Error(), "unexpected type: 0")
 }
 
@@ -57,9 +56,9 @@ func TestDecoderWhenYAMLHasInvalidKeys(t *testing.T) {
 		math.NaN(): "",
 	}}
 	decoder := &yaml.Decoder{
-		Unmarshal:  mock.unmarshal,
+		DecodeYAML: mock.decode,
 		KeyMarshal: json.Marshal,
 	}
-	_, err := decoder.JSON(nil)
+	_, err := decoder.JSON()
 	assertEq(t, err.Error(), "json: unsupported value: NaN")
 }

yaml/encoder.go

@@ -7,7 +7,7 @@ import (
 )
 
 type Encoder struct {
-	Marshal      func(interface{}) ([]byte, error)
+	EncodeYAML   func(interface{}) error
 	KeyUnmarshal func([]byte, interface{}) error
 
 	// If set, these will be converted to floats.
@@ -18,9 +18,9 @@ type Encoder struct {
 // Special string keys from the Decoder are accounted for.
 // YAML objects are accepted, as long as represent valid JSON.
 // Internal structs are currently passed through unmodified.
-func (e *Encoder) YAML(json interface{}) (yaml []byte, err error) {
+func (e *Encoder) YAML(json interface{}) (err error) {
 	defer catchFailure(&err)
-	return e.Marshal(e.yamlify(json))
+	return e.EncodeYAML(e.yamlify(json))
 }
 
 func (e *Encoder) yamlify(in interface{}) interface{} {

yaml/encoder_test.go

@@ -11,41 +11,40 @@ import (
 )
 
 func TestEncoder(t *testing.T) {
-	mock := &mockYAML{data: []byte("some YAML")}
+	mock := &mockYAML{}
 	encoder := &yaml.Encoder{
-		Marshal:      mock.marshal,
+		EncodeYAML:   mock.encode,
 		KeyUnmarshal: keyUnmarshal,
 		NaN:          F{"NaN"},
 		PosInf:       F{"Infinity"},
 		NegInf:       F{"-Infinity"},
 	}
-	yaml, err := encoder.YAML(jsonFixture)
+	err := encoder.YAML(jsonFixture)
 	assertEq(t, err, nil)
 	assertEq(t, mock.value, yamlFixture)
-	assertEq(t, yaml, []byte("some YAML"))
 }
 
 func TestEncoderWhenYAMLIsInvalid(t *testing.T) {
 	mock := &mockYAML{err: errors.New("some error")}
-	encoder := &yaml.Encoder{Marshal: mock.marshal}
-	_, err := encoder.YAML(nil)
+	encoder := &yaml.Encoder{EncodeYAML: mock.encode}
+	err := encoder.YAML(nil)
 	assertEq(t, err.Error(), "some error")
 }
 
 func TestEncoderWhenYAMLHasInvalidTypes(t *testing.T) {
 	mock := &mockYAML{}
-	encoder := &yaml.Encoder{Marshal: mock.marshal}
+	encoder := &yaml.Encoder{EncodeYAML: mock.encode}
 
-	_, err := encoder.YAML(map[int]int{})
+	err := encoder.YAML(map[int]int{})
 	assertEq(t, err.Error(), "unexpected type: map[int]int{}")
 
-	_, err = encoder.YAML([0]int{})
+	err = encoder.YAML([0]int{})
 	assertEq(t, err.Error(), "unexpected type: [0]int{}")
 
-	_, err = encoder.YAML([]int{})
+	err = encoder.YAML([]int{})
 	assertEq(t, err.Error(), "unexpected type: []int{}")
 
-	_, err = encoder.YAML(float32(0))
+	err = encoder.YAML(float32(0))
 	assertEq(t, err.Error(), "unexpected type: 0")
 }