Skip to content

Commit

Permalink
feat: support zstd compression
Browse files Browse the repository at this point in the history
  • Loading branch information
vicanso committed Nov 7, 2023
1 parent 03cd7b7 commit 7aab359
Show file tree
Hide file tree
Showing 9 changed files with 173 additions and 19 deletions.
2 changes: 2 additions & 0 deletions df.go
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,8 @@ const (
Gzip = "gzip"
// Br brotli compress
Br = "br"
// Zstd zstd compress
Zstd = "zstd"
)

var (
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ go 1.18
require (
github.com/andybalholm/brotli v1.0.6
github.com/hashicorp/golang-lru/v2 v2.0.7
github.com/klauspost/compress v1.17.2
github.com/stretchr/testify v1.8.4
github.com/tidwall/gjson v1.17.0
github.com/vicanso/hes v0.6.2
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
Expand Down
3 changes: 1 addition & 2 deletions middleware/brotli.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@ import (

const (
// BrEncoding br encoding
BrEncoding = "br"
maxBrQuality = 11
defaultBrQuality = 6
)
Expand Down Expand Up @@ -69,7 +68,7 @@ func (b *BrCompressor) Accept(c *elton.Context, bodySize int) (acceptable bool,
if bodySize >= 0 && bodySize < b.getMinLength() {
return
}
return AcceptEncoding(c, BrEncoding)
return AcceptEncoding(c, elton.Br)
}

// BrotliCompress compress data by brotli
Expand Down
45 changes: 40 additions & 5 deletions middleware/cache_compressor.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,9 @@ import (
"bytes"
"compress/gzip"
"regexp"

"github.com/klauspost/compress/zstd"
"github.com/vicanso/elton"
)

type CompressionType uint8
Expand All @@ -37,6 +40,8 @@ const (
CompressionGzip
// br compress
CompressionBr
// zstd compress
CompressionZstd
)

type CacheCompressor interface {
Expand Down Expand Up @@ -81,18 +86,17 @@ func (br *CacheBrCompressor) Decompress(data *bytes.Buffer) (*bytes.Buffer, erro
return BrotliDecompress(data.Bytes())
}
func (br *CacheBrCompressor) GetEncoding() string {
return BrEncoding
return elton.Br
}
func (br *CacheBrCompressor) IsValid(contentType string, length int) bool {
return isValidForCompress(br.ContentRegexp, br.MinLength, contentType, length)
}

func (br *CacheBrCompressor) Compress(buffer *bytes.Buffer) (*bytes.Buffer, CompressionType, error) {
data, err := BrotliCompress(buffer.Bytes(), br.Level)
if err != nil {
return nil, CompressionNone, err
}
return data, CompressionBr, nil
return data, br.GetCompression(), nil
}
func (br *CacheBrCompressor) GetCompression() CompressionType {
return CompressionBr
Expand All @@ -114,7 +118,7 @@ func (g *CacheGzipCompressor) Decompress(data *bytes.Buffer) (*bytes.Buffer, err
return GzipDecompress(data.Bytes())
}
func (g *CacheGzipCompressor) GetEncoding() string {
return GzipEncoding
return elton.Gzip
}
func (g *CacheGzipCompressor) IsValid(contentType string, length int) bool {
return isValidForCompress(g.ContentRegexp, g.MinLength, contentType, length)
Expand All @@ -124,8 +128,39 @@ func (g *CacheGzipCompressor) Compress(buffer *bytes.Buffer) (*bytes.Buffer, Com
if err != nil {
return nil, CompressionNone, err
}
return data, CompressionGzip, nil
return data, g.GetCompression(), nil
}
func (g *CacheGzipCompressor) GetCompression() CompressionType {
return CompressionGzip
}

type CacheZstdCompressor struct {
Level int
MinLength int
ContentRegexp *regexp.Regexp
}

func NewCacheZstdCompressor() *CacheZstdCompressor {
return &CacheZstdCompressor{
Level: int(zstd.SpeedBetterCompression),
}
}
func (z *CacheZstdCompressor) Decompress(data *bytes.Buffer) (*bytes.Buffer, error) {
return ZstdDecompress(data.Bytes())
}
func (z *CacheZstdCompressor) GetEncoding() string {
return elton.Zstd
}
func (z *CacheZstdCompressor) IsValid(contentType string, length int) bool {
return isValidForCompress(z.ContentRegexp, z.MinLength, contentType, length)
}
func (z *CacheZstdCompressor) Compress(buffer *bytes.Buffer) (*bytes.Buffer, CompressionType, error) {
data, err := ZstdCompress(buffer.Bytes(), z.Level)
if err != nil {
return nil, CompressionNone, err
}
return data, z.GetCompression(), nil
}
func (z *CacheZstdCompressor) GetCompression() CompressionType {
return CompressionZstd
}
8 changes: 4 additions & 4 deletions middleware/cache_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -177,8 +177,8 @@ func TestCacheResponseGetBody(t *testing.T) {
}
},
compressor: NewCacheBrCompressor(),
acceptEncoding: BrEncoding,
encoding: BrEncoding,
acceptEncoding: elton.Br,
encoding: elton.Br,
body: brData,
},
// 数据br, 客户端不支持br
Expand All @@ -203,8 +203,8 @@ func TestCacheResponseGetBody(t *testing.T) {
}
},
compressor: NewCacheGzipCompressor(),
acceptEncoding: GzipEncoding,
encoding: GzipEncoding,
acceptEncoding: elton.Gzip,
encoding: elton.Gzip,
body: gzipData,
},
// 数据gzip,客户端不支持gzip
Expand Down
4 changes: 2 additions & 2 deletions middleware/compressor_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ func TestCompressor(t *testing.T) {
}{
{
compressor: new(GzipCompressor),
encoding: GzipEncoding,
encoding: elton.Gzip,
uncompress: func(b []byte) ([]byte, error) {
buffer, err := GzipDecompress(b)
if err != nil {
Expand All @@ -53,7 +53,7 @@ func TestCompressor(t *testing.T) {
},
{
compressor: new(BrCompressor),
encoding: BrEncoding,
encoding: elton.Br,
uncompress: func(b []byte) ([]byte, error) {
buffer, err := BrotliDecompress(b)
if err != nil {
Expand Down
7 changes: 1 addition & 6 deletions middleware/gzip.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,6 @@ import (
"github.com/vicanso/elton"
)

const (
// GzipEncoding gzip encoding
GzipEncoding = "gzip"
)

type (
// GzipCompressor gzip compress
GzipCompressor struct {
Expand All @@ -49,7 +44,7 @@ func (g *GzipCompressor) Accept(c *elton.Context, bodySize int) (bool, string) {
if bodySize >= 0 && bodySize < g.getMinLength() {
return false, ""
}
return AcceptEncoding(c, GzipEncoding)
return AcceptEncoding(c, elton.Gzip)
}

// GzipCompress compress data by gzip
Expand Down
120 changes: 120 additions & 0 deletions middleware/zstd.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,120 @@
// MIT License

// Copyright (c) 2023 Tree Xie

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:

// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.

// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.

package middleware

import (
"bytes"
"io"

"github.com/klauspost/compress/zstd"
"github.com/vicanso/elton"
)

type (
// ZstdCompressor zstd compress
ZstdCompressor struct {
Level int
MinLength int
}
)

// Accept accept zstd encoding
func (z *ZstdCompressor) Accept(c *elton.Context, bodySize int) (bool, string) {

// 如果数据少于最低压缩长度,则不压缩(因为reader中的bodySize会设置为1,因此需要判断>=0)
if bodySize >= 0 && bodySize < z.getMinLength() {
return false, ""
}
return AcceptEncoding(c, elton.Zstd)
}

func (z *ZstdCompressor) getLevel() int {
level := z.Level
if level <= 0 {
level = int(zstd.SpeedBetterCompression)
}
if level > int(zstd.SpeedBestCompression) {
level = int(zstd.SpeedBestCompression)
}
return level
}

func (z *ZstdCompressor) getMinLength() int {
if z.MinLength == 0 {
return DefaultCompressMinLength
}
return z.MinLength
}

// Compress compress data by zstd
func (z *ZstdCompressor) Compress(buf []byte, levels ...int) (*bytes.Buffer, error) {
level := z.getLevel()
if len(levels) != 0 && levels[0] != IgnoreCompression {
level = levels[0]
}
return GzipCompress(buf, level)
}

// Pipe compress by pipe
func (z *ZstdCompressor) Pipe(c *elton.Context) error {
r := c.Body.(io.Reader)
closer, ok := c.Body.(io.Closer)
if ok {
defer closer.Close()
}
enc, err := zstd.NewWriter(c.Response, zstd.WithEncoderLevel(zstd.EncoderLevel(z.getLevel())))
if err != nil {
return err
}

_, err = io.Copy(enc, r)
if err != nil {
enc.Close()
return err
}
return enc.Close()
}

// ZstdCompressor compress data by zstd
func ZstdCompress(buf []byte, level int) (*bytes.Buffer, error) {
encoder, err := zstd.NewWriter(nil, zstd.WithEncoderLevel(zstd.EncoderLevel(level)))
if err != nil {
return nil, err
}
dst := encoder.EncodeAll(buf, make([]byte, 0, len(buf)))
return bytes.NewBuffer(dst), nil
}

// ZstdDecompress decompress data of zstd
func ZstdDecompress(buf []byte) (*bytes.Buffer, error) {
r, err := zstd.NewReader(bytes.NewBuffer(buf))
if err != nil {
return nil, err
}
defer r.Close()
data, err := io.ReadAll(r)
if err != nil {
return nil, err
}
return bytes.NewBuffer(data), nil
}

0 comments on commit 7aab359

Please sign in to comment.