Content decompressors and go min version update

.github/workflows/ci.yml

@@ -25,7 +25,7 @@ jobs:
     name: Build
     strategy:
       matrix:
-        go: [ 'stable', '1.20.x' ]
+        go: [ 'stable', '1.21.x' ]
         os: [ ubuntu-latest ]
 
     runs-on: ${{ matrix.os }}
@@ -47,9 +47,12 @@ jobs:
         run: diff -u <(echo -n) <(go fmt $(go list ./...))
 
       - name: Test
-        run: go test ./... -race -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...
+        run: go test ./... -race -count=1 -coverprofile=coverage.txt -covermode=atomic -coverpkg=./... -shuffle=on
 
       - name: Upload coverage to Codecov
+        if: ${{ matrix.os == 'ubuntu-latest' && matrix.go == 'stable' }}
         uses: codecov/codecov-action@v4
-        env:
-          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          file: ./coverage.txt
+          flags: unittests

.github/workflows/label-actions.yml

@@ -12,7 +12,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go: [ 'stable', '1.20.x' ]
+        go: [ 'stable', '1.21.x' ]
         os: [ ubuntu-latest ]
 
     name: Run Build
@@ -36,9 +36,12 @@ jobs:
         run: diff -u <(echo -n) <(go fmt $(go list ./...))
 
       - name: Test
-        run: go test ./... -race -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...
+        run: go test ./... -race -count=1 -coverprofile=coverage.txt -covermode=atomic -coverpkg=./... -shuffle=on
 
       - name: Upload coverage to Codecov
+        if: ${{ matrix.os == 'ubuntu-latest' && matrix.go == 'stable' }}
         uses: codecov/codecov-action@v4
-        env:
-          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          file: ./coverage.txt
+          flags: unittests

client.go

@@ -15,6 +15,7 @@ import (
 	"net/url"
 	"os"
 	"reflect"
+	"slices"
 	"strings"
 	"sync"
 	"time"
@@ -52,6 +53,7 @@ const (
 var (
 	hdrUserAgentKey       = http.CanonicalHeaderKey("User-Agent")
 	hdrAcceptKey          = http.CanonicalHeaderKey("Accept")
+	hdrAcceptEncodingKey  = http.CanonicalHeaderKey("Accept-Encoding")
 	hdrContentTypeKey     = http.CanonicalHeaderKey("Content-Type")
 	hdrContentLengthKey   = http.CanonicalHeaderKey("Content-Length")
 	hdrContentEncodingKey = http.CanonicalHeaderKey("Content-Encoding")
@@ -128,9 +130,6 @@ type TransportSettings struct {
 	// DisableKeepAlives, default value is `false`.
 	DisableKeepAlives bool
 
-	// DisableCompression, default value is `false`.
-	DisableCompression bool
-
 	// MaxResponseHeaderBytes, added to provide ability to
 	// set value. No default value in Resty, the Go
 	// HTTP client default value applies.
@@ -153,55 +152,57 @@ type TransportSettings struct {
 // Resty also provides an option to override most of the client settings
 // at [Request] level.
 type Client struct {
-	lock                   *sync.RWMutex
-	baseURL                string
-	queryParams            url.Values
-	formData               url.Values
-	pathParams             map[string]string
-	rawPathParams          map[string]string
-	header                 http.Header
-	userInfo               *User
-	authToken              string
-	authScheme             string
-	cookies                []*http.Cookie
-	errorType              reflect.Type
-	debug                  bool
-	disableWarn            bool
-	allowGetMethodPayload  bool
-	retryCount             int
-	retryWaitTime          time.Duration
-	retryMaxWaitTime       time.Duration
-	retryConditions        []RetryConditionFunc
-	retryHooks             []OnRetryFunc
-	retryAfter             RetryAfterFunc
-	retryResetReaders      bool
-	headerAuthorizationKey string
-	responseBodyLimit      int64
-	resBodyUnlimitedReads  bool
-	jsonEscapeHTML         bool
-	setContentLength       bool
-	closeConnection        bool
-	notParseResponse       bool
-	isTrace                bool
-	debugBodyLimit         int
-	outputDirectory        string
-	scheme                 string
-	log                    Logger
-	httpClient             *http.Client
-	proxyURL               *url.URL
-	requestLog             RequestLogCallback
-	responseLog            ResponseLogCallback
-	rateLimiter            RateLimiter
-	generateCurlOnDebug    bool
-	beforeRequest          []RequestMiddleware
-	udBeforeRequest        []RequestMiddleware
-	afterResponse          []ResponseMiddleware
-	errorHooks             []ErrorHook
-	invalidHooks           []ErrorHook
-	panicHooks             []ErrorHook
-	successHooks           []SuccessHook
-	contentTypeEncoders    map[string]ContentTypeEncoder
-	contentTypeDecoders    map[string]ContentTypeDecoder
+	lock                    *sync.RWMutex
+	baseURL                 string
+	queryParams             url.Values
+	formData                url.Values
+	pathParams              map[string]string
+	rawPathParams           map[string]string
+	header                  http.Header
+	userInfo                *User
+	authToken               string
+	authScheme              string
+	cookies                 []*http.Cookie
+	errorType               reflect.Type
+	debug                   bool
+	disableWarn             bool
+	allowGetMethodPayload   bool
+	retryCount              int
+	retryWaitTime           time.Duration
+	retryMaxWaitTime        time.Duration
+	retryConditions         []RetryConditionFunc
+	retryHooks              []OnRetryFunc
+	retryAfter              RetryAfterFunc
+	retryResetReaders       bool
+	headerAuthorizationKey  string
+	responseBodyLimit       int64
+	resBodyUnlimitedReads   bool
+	jsonEscapeHTML          bool
+	setContentLength        bool
+	closeConnection         bool
+	notParseResponse        bool
+	isTrace                 bool
+	debugBodyLimit          int
+	outputDirectory         string
+	scheme                  string
+	log                     Logger
+	httpClient              *http.Client
+	proxyURL                *url.URL
+	requestLog              RequestLogCallback
+	responseLog             ResponseLogCallback
+	rateLimiter             RateLimiter
+	generateCurlOnDebug     bool
+	beforeRequest           []RequestMiddleware
+	udBeforeRequest         []RequestMiddleware
+	afterResponse           []ResponseMiddleware
+	errorHooks              []ErrorHook
+	invalidHooks            []ErrorHook
+	panicHooks              []ErrorHook
+	successHooks            []SuccessHook
+	contentTypeEncoders     map[string]ContentTypeEncoder
+	contentTypeDecoders     map[string]ContentTypeDecoder
+	contentDecompressorKeys []string
+	contentDecompressors    map[string]ContentDecompressor
 
 	// TODO don't put mutex now, it may go away
 	preReqHook PreRequestHook
@@ -776,6 +777,70 @@ func (c *Client) inferContentTypeDecoder(ct ...string) (ContentTypeDecoder, bool
 	return nil, false
 }
 
+// ContentDecompressors method returns all the registered content-encoding decompressors.
+func (c *Client) ContentDecompressors() map[string]ContentDecompressor {
+	c.lock.RLock()
+	defer c.lock.RUnlock()
+	return c.contentDecompressors
+}
+
+// AddContentDecompressor method adds the user-provided Content-Encoding ([RFC 9110]) decompressor
+// and directive into a client.
+//
+// NOTE: It overwrites the decompressor function if the given Content-Encoding directive already exists.
+//
+// [RFC 9110]: https://datatracker.ietf.org/doc/html/rfc9110
+func (c *Client) AddContentDecompressor(k string, d ContentDecompressor) *Client {
+	c.insertFirstContentDecompressor(k)
+
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	c.contentDecompressors[k] = d
+	return c
+}
+
+// ContentDecompressorKeys method returns all the registered content-encoding decompressors
+// keys as comma-separated string.
+func (c *Client) ContentDecompressorKeys() string {
+	c.lock.RLock()
+	defer c.lock.RUnlock()
+	return strings.Join(c.contentDecompressorKeys, ", ")
+}
+
+// SetContentDecompressorKeys method sets given Content-Encoding ([RFC 9110]) directives into the client instance.
+//
+// It checks the given Content-Encoding exists in the [ContentDecompressor] list before assigning it,
+// if it does not exist, it will skip that directive.
+//
+// Use this method to overwrite the default order. If a new content decompressor is added,
+// that directive will be the first.
+//
+// [RFC 9110]: https://datatracker.ietf.org/doc/html/rfc9110
+func (c *Client) SetContentDecompressorKeys(keys []string) *Client {
+	result := make([]string, 0)
+	decoders := c.ContentDecompressors()
+	for _, k := range keys {
+		if _, f := decoders[k]; f {
+			result = append(result, k)
+		}
+	}
+
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	c.contentDecompressorKeys = result
+	return c
+}
+
+func (c *Client) insertFirstContentDecompressor(k string) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	if !slices.Contains(c.contentDecompressorKeys, k) {
+		c.contentDecompressorKeys = append(c.contentDecompressorKeys, "")
+		copy(c.contentDecompressorKeys[1:], c.contentDecompressorKeys)
+		c.contentDecompressorKeys[0] = k
+	}
+}
+
 // IsDebug method returns `true` if the client is in debug mode; otherwise, it is `false`.
 func (c *Client) IsDebug() bool {
 	c.lock.RLock()
@@ -1741,10 +1806,14 @@ func (c *Client) execute(req *Request) (*Response, error) {
 		return response, err
 	}
 	if resp != nil {
-		response.Body = &limitReadCloser{r: resp.Body,
-			l: req.ResponseBodyLimit,
-			f: func(s int64) { response.size = s },
+		response.Body = resp.Body
+
+		err := response.wrapContentDecompressor()
+		if err != nil {
+			return response, err
 		}
+
+		response.wrapLimitReadCloser()
 	}
 	if !req.DoNotParseResponse && (req.Debug || req.ResponseBodyUnlimitedReads) {
 		response.wrapReadCopier()

client_test.go

@@ -7,6 +7,7 @@ package resty
 import (
 	"bytes"
 	"compress/gzip"
+	"compress/lzw"
 	"crypto/rand"
 	"crypto/tls"
 	"errors"
@@ -644,20 +645,95 @@ func (rt *CustomRoundTripper) RoundTrip(_ *http.Request) (*http.Response, error)
 	return &http.Response{}, nil
 }
 
-func TestAutoGzip(t *testing.T) {
+func TestGzipCompress(t *testing.T) {
 	ts := createGenericServer(t)
 	defer ts.Close()
 
-	c := New()
+	c := dcnl()
 	testcases := []struct{ url, want string }{
 		{ts.URL + "/gzip-test", "This is Gzip response testing"},
 		{ts.URL + "/gzip-test-gziped-empty-body", ""},
 		{ts.URL + "/gzip-test-no-gziped-body", ""},
 	}
 	for _, tc := range testcases {
-		resp, err := c.R().
-			// SetHeader("Accept-Encoding", "gzip"). // TODO put back when implementing compression handling
-			Get(tc.url)
+		resp, err := c.R().Get(tc.url)
+
+		assertError(t, err)
+		assertEqual(t, http.StatusOK, resp.StatusCode())
+		assertEqual(t, "200 OK", resp.Status())
+		assertNotNil(t, resp.BodyBytes())
+		assertEqual(t, tc.want, resp.String())
+
+		logResponse(t, resp)
+	}
+}
+
+func TestDeflateCompress(t *testing.T) {
+	ts := createGenericServer(t)
+	defer ts.Close()
+
+	c := dcnl()
+	testcases := []struct{ url, want string }{
+		{ts.URL + "/deflate-test", "This is Deflate response testing"},
+		{ts.URL + "/deflate-test-empty-body", ""},
+		{ts.URL + "/deflate-test-no-body", ""},
+	}
+	for _, tc := range testcases {
+		resp, err := c.R().Get(tc.url)
+
+		assertError(t, err)
+		assertEqual(t, http.StatusOK, resp.StatusCode())
+		assertEqual(t, "200 OK", resp.Status())
+		assertNotNil(t, resp.BodyBytes())
+		assertEqual(t, tc.want, resp.String())
+
+		logResponse(t, resp)
+	}
+}
+
+type lzwReader struct {
+	s io.ReadCloser
+	r io.ReadCloser
+}
+
+func (l *lzwReader) Read(p []byte) (n int, err error) {
+	return l.r.Read(p)
+}
+
+func (l *lzwReader) Close() error {
+	closeq(l.r)
+	closeq(l.s)
+	return nil
+}
+
+func TestLzwCompress(t *testing.T) {
+	ts := createGenericServer(t)
+	defer ts.Close()
+
+	c := dcnl()
+
+	// Not found scenario
+	_, err := c.R().Get(ts.URL + "/lzw-test")
+	assertNotNil(t, err)
+	assertEqual(t, ErrContentDecompressorNotFound, err)
+
+	// Register LZW content decoder
+	c.AddContentDecompressor("compress", func(r io.ReadCloser) (io.ReadCloser, error) {
+		l := &lzwReader{
+			s: r,
+			r: lzw.NewReader(r, lzw.LSB, 8),
+		}
+		return l, nil
+	})
+	c.SetContentDecompressorKeys([]string{"compress"})
+
+	testcases := []struct{ url, want string }{
+		{ts.URL + "/lzw-test", "This is LZW response testing"},
+		{ts.URL + "/lzw-test-empty-body", ""},
+		{ts.URL + "/lzw-test-no-body", ""},
+	}
+	for _, tc := range testcases {
+		resp, err := c.R().Get(tc.url)
 
 		assertError(t, err)
 		assertEqual(t, http.StatusOK, resp.StatusCode())

go.mod

@@ -1,6 +1,6 @@
 module github.com/go-resty/resty/v3
 
-go 1.20
+go 1.21
 
 require (
 	golang.org/x/net v0.27.0

middleware.go

@@ -170,6 +170,10 @@ func parseRequestHeader(c *Client, r *Request) error {
 		r.Header.Set(hdrAcceptKey, r.Header.Get(hdrContentTypeKey))
 	}
 
+	if isStringEmpty(r.Header.Get(hdrAcceptEncodingKey)) {
+		r.Header.Set(hdrAcceptEncodingKey, r.client.ContentDecompressorKeys())
+	}
+
 	return nil
 }