From 245ff11191d82c0dc8f8f86e16cc57c75c073368 Mon Sep 17 00:00:00 2001 From: wenweihuang Date: Thu, 19 Dec 2024 17:22:12 +0800 Subject: [PATCH] feat(outputs): Add InLong output plugin --- docs/LICENSE_OF_DEPENDENCIES.md | 8 +++ go.mod | 8 +++ go.sum | 24 +++++++- plugins/outputs/all/inlong.go | 5 ++ plugins/outputs/inlong/README.md | 44 +++++++++++++ plugins/outputs/inlong/inlong.go | 89 +++++++++++++++++++++++++++ plugins/outputs/inlong/inlong_test.go | 68 ++++++++++++++++++++ plugins/outputs/inlong/sample.conf | 27 ++++++++ 8 files changed, 270 insertions(+), 3 deletions(-) create mode 100644 plugins/outputs/all/inlong.go create mode 100644 plugins/outputs/inlong/README.md create mode 100644 plugins/outputs/inlong/inlong.go create mode 100644 plugins/outputs/inlong/inlong_test.go create mode 100644 plugins/outputs/inlong/sample.conf diff --git a/docs/LICENSE_OF_DEPENDENCIES.md b/docs/LICENSE_OF_DEPENDENCIES.md index b8994365b6154..bee75fe5e96f0 100644 --- a/docs/LICENSE_OF_DEPENDENCIES.md +++ b/docs/LICENSE_OF_DEPENDENCIES.md @@ -55,6 +55,7 @@ following works: - github.com/antchfx/xpath [MIT License](https://github.com/antchfx/xpath/blob/master/LICENSE) - github.com/antlr4-go/antlr [BSD 3-Clause "New" or "Revised" License](https://github.com/antlr/antlr4/blob/master/LICENSE.txt) - github.com/apache/arrow/go [Apache License 2.0](https://github.com/apache/arrow/blob/master/LICENSE.txt) +- github.com/apache/inlong/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang [Apache License 2.0](https://github.com/apache/inlong/blob/master/LICENSE) - github.com/apache/iotdb-client-go [Apache License 2.0](https://github.com/apache/iotdb-client-go/blob/main/LICENSE) - github.com/apache/thrift [Apache License 2.0](https://github.com/apache/thrift/blob/master/LICENSE) - github.com/aristanetworks/glog [Apache License 2.0](https://github.com/aristanetworks/glog/blob/master/LICENSE) @@ -97,6 +98,7 @@ following works: - github.com/bmatcuk/doublestar [MIT License](https://github.com/bmatcuk/doublestar/blob/master/LICENSE) - github.com/boschrexroth/ctrlx-datalayer-golang [MIT License](https://github.com/boschrexroth/ctrlx-datalayer-golang/blob/main/LICENSE) - github.com/bufbuild/protocompile [Apache License 2.0](https://github.com/bufbuild/protocompile/blob/main/LICENSE) +- github.com/bwmarrin/snowflake [BSD 2-Clause "Simplified" License](https://github.com/bwmarrin/snowflake/blob/master/LICENSE) - github.com/caio/go-tdigest [MIT License](https://github.com/caio/go-tdigest/blob/master/LICENSE) - github.com/cenkalti/backoff [MIT License](https://github.com/cenkalti/backoff/blob/master/LICENSE) - github.com/census-instrumentation/opencensus-proto [Apache License 2.0](https://github.com/census-instrumentation/opencensus-proto/blob/master/LICENSE) @@ -157,6 +159,7 @@ following works: - github.com/go-openapi/jsonreference [Apache License 2.0](https://github.com/go-openapi/jsonreference/blob/master/LICENSE) - github.com/go-openapi/swag [Apache License 2.0](https://github.com/go-openapi/swag/blob/master/LICENSE) - github.com/go-redis/redis [BSD 2-Clause "Simplified" License](https://github.com/go-redis/redis/blob/master/LICENSE) +- github.com/go-resty/resty [MIT License](https://github.com/go-resty/resty/blob/v2/LICENSE) - github.com/go-sql-driver/mysql [Mozilla Public License 2.0](https://github.com/go-sql-driver/mysql/blob/master/LICENSE) - github.com/go-stack/stack [MIT License](https://github.com/go-stack/stack/blob/master/LICENSE.md) - github.com/go-stomp/stomp [Apache License 2.0](https://github.com/go-stomp/stomp/blob/master/LICENSE.txt) @@ -309,7 +312,9 @@ following works: - github.com/opencontainers/image-spec [Apache License 2.0](https://github.com/opencontainers/image-spec/blob/master/LICENSE) - github.com/opensearch-project/opensearch-go [Apache License 2.0](https://github.com/opensearch-project/opensearch-go/blob/main/LICENSE.txt) - github.com/opentracing/opentracing-go [Apache License 2.0](https://github.com/opentracing/opentracing-go/blob/master/LICENSE) +- github.com/oxtoacart/bpool [Apache License 2.0](https://github.com/oxtoacart/bpool/blob/master/LICENSE) - github.com/p4lang/p4runtime [Apache License 2.0](https://github.com/p4lang/p4runtime/blob/main/LICENSE) +- github.com/panjf2000/gnet [Apache License 2.0](https://github.com/panjf2000/gnet/blob/dev/LICENSE) - github.com/pborman/ansi [BSD 3-Clause "New" or "Revised" License](https://github.com/pborman/ansi/blob/master/LICENSE) - github.com/pcolladosoto/goslurm [MIT License](https://github.com/pcolladosoto/goslurm/blob/main/LICENSE) - github.com/peterbourgon/unixtransport [Apache License 2.0](https://github.com/peterbourgon/unixtransport/blob/main/LICENSE) @@ -376,6 +381,7 @@ following works: - github.com/uber/jaeger-client-go [Apache License 2.0](https://github.com/jaegertracing/jaeger-client-go/blob/master/LICENSE) - github.com/uber/jaeger-lib [Apache License 2.0](https://github.com/jaegertracing/jaeger-lib/blob/main/LICENSE) - github.com/urfave/cli [MIT License](https://github.com/urfave/cli/blob/main/LICENSE) +- github.com/valyala/bytebufferpool [MIT License](https://github.com/valyala/bytebufferpool/blob/master/LICENSE) - github.com/vapourismo/knx-go [MIT License](https://github.com/vapourismo/knx-go/blob/master/LICENSE) - github.com/vishvananda/netlink [Apache License 2.0](https://github.com/vishvananda/netlink/blob/master/LICENSE) - github.com/vishvananda/netns [Apache License 2.0](https://github.com/vishvananda/netns/blob/master/LICENSE) @@ -394,6 +400,7 @@ following works: - github.com/yuin/gopher-lua [MIT License](https://github.com/yuin/gopher-lua/blob/master/LICENSE) - github.com/yusufpapurcu/wmi [MIT License](https://github.com/yusufpapurcu/wmi/blob/master/LICENSE) - github.com/zeebo/xxh3 [BSD 2-Clause "Simplified" License](https://github.com/zeebo/xxh3/blob/master/LICENSE) +- github.com/zentures/cityhash [MIT License](https://github.com/zentures/cityhash/blob/master/LICENSE) - go.mongodb.org/mongo-driver [Apache License 2.0](https://github.com/mongodb/mongo-go-driver/blob/master/LICENSE) - go.opencensus.io [Apache License 2.0](https://github.com/census-instrumentation/opencensus-go/blob/master/LICENSE) - go.opentelemetry.io/collector/consumer [Apache License 2.0](https://github.com/open-telemetry/opentelemetry-collector/blob/main/LICENSE) @@ -438,6 +445,7 @@ following works: - gopkg.in/gorethink/gorethink.v3 [Apache License 2.0](https://github.com/rethinkdb/rethinkdb-go/blob/v3.0.5/LICENSE) - gopkg.in/inf.v0 [BSD 3-Clause "New" or "Revised" License](https://github.com/go-inf/inf/blob/v0.9.1/LICENSE) - gopkg.in/ini.v1 [Apache License 2.0](https://github.com/go-ini/ini/blob/master/LICENSE) +- gopkg.in/natefinch/lumberjack.v2 [MIT License](https://github.com/natefinch/lumberjack/blob/v2.0/LICENSE) - gopkg.in/olivere/elastic.v5 [MIT License](https://github.com/olivere/elastic/blob/v5.0.76/LICENSE) - gopkg.in/tomb.v1 [BSD 3-Clause Clear License](https://github.com/go-tomb/tomb/blob/v1/LICENSE) - gopkg.in/tomb.v2 [BSD 3-Clause Clear License](https://github.com/go-tomb/tomb/blob/v2/LICENSE) diff --git a/go.mod b/go.mod index fd1daea38d548..5570ab00f3c5b 100644 --- a/go.mod +++ b/go.mod @@ -39,6 +39,7 @@ require ( github.com/antchfx/xmlquery v1.4.1 github.com/antchfx/xpath v1.3.1 github.com/apache/arrow/go/v18 v18.0.0-20240716144821-cf5d7c7ec3cf + github.com/apache/inlong/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang v0.0.0-20241120061539-3700baa19f68 github.com/apache/iotdb-client-go v1.3.2 github.com/apache/thrift v0.21.0 github.com/aristanetworks/goarista v0.0.0-20190325233358-a123909ec740 @@ -301,6 +302,7 @@ require ( github.com/bitly/go-hostpool v0.1.0 // indirect github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect github.com/bufbuild/protocompile v0.10.0 // indirect + github.com/bwmarrin/snowflake v0.3.0 // indirect github.com/caio/go-tdigest/v4 v4.0.1 // indirect github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cenkalti/backoff/v4 v4.3.0 // indirect @@ -342,6 +344,7 @@ require ( github.com/go-openapi/jsonpointer v0.20.2 // indirect github.com/go-openapi/jsonreference v0.20.4 // indirect github.com/go-openapi/swag v0.22.9 // indirect + github.com/go-resty/resty/v2 v2.13.1 // indirect github.com/go-stack/stack v1.8.1 // indirect github.com/goburrow/modbus v0.1.0 // indirect github.com/goburrow/serial v0.1.1-0.20211022031912-bfb69110f8dd // indirect @@ -442,6 +445,8 @@ require ( github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492 // indirect + github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c // indirect + github.com/panjf2000/gnet/v2 v2.5.7 // indirect github.com/philhofer/fwd v1.1.3-0.20240612014219-fbbf4953d986 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pion/logging v0.2.2 // indirect @@ -479,6 +484,7 @@ require ( github.com/twmb/murmur3 v1.1.7 // indirect github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect github.com/uber/jaeger-lib v2.4.1+incompatible // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/xdg-go/pbkdf2 v1.0.0 // indirect github.com/xdg-go/scram v1.1.2 // indirect @@ -490,6 +496,7 @@ require ( github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/zeebo/assert v1.3.1 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect + github.com/zentures/cityhash v0.0.0-20131128155616-cdd6a94144ab // indirect github.com/zitadel/logging v0.6.1 // indirect github.com/zitadel/oidc/v3 v3.30.0 // indirect github.com/zitadel/schema v1.3.0 // indirect @@ -519,6 +526,7 @@ require ( gopkg.in/fsnotify.v1 v1.4.7 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/sourcemap.v1 v1.0.5 // indirect gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index b982de44cc6b4..4c02177e789b0 100644 --- a/go.sum +++ b/go.sum @@ -836,6 +836,8 @@ github.com/apache/arrow/go/v15 v15.0.2 h1:60IliRbiyTWCWjERBCkO1W4Qun9svcYoZrSLcy github.com/apache/arrow/go/v15 v15.0.2/go.mod h1:DGXsR3ajT524njufqf95822i+KTh+yea1jass9YXgjA= github.com/apache/arrow/go/v18 v18.0.0-20240716144821-cf5d7c7ec3cf h1:9b4bG4uqvid0RH3MHWq2soXTfhPFbqbuNCqLRrl4ZGg= github.com/apache/arrow/go/v18 v18.0.0-20240716144821-cf5d7c7ec3cf/go.mod h1:84kVJOfdiXAj9Zo8lvZ2uuJVzPn2vKlPdrSHU1zD2mE= +github.com/apache/inlong/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang v0.0.0-20241120061539-3700baa19f68 h1:IaHtzTmjOUvGo2JxGmDVFVwCFnphlwaFmWv3kfrCC9M= +github.com/apache/inlong/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang v0.0.0-20241120061539-3700baa19f68/go.mod h1:d+xqug/5+N0HnQVDf+0gTyB/SFIGjD3VpCbTalvBXzM= github.com/apache/iotdb-client-go v1.3.2 h1:IPPVlOganGJ6Q0NTWtktLgsvsuG9YIRP1U6nhO9ee6k= github.com/apache/iotdb-client-go v1.3.2/go.mod h1:3D6QYkqRmASS/4HsjU+U/3fscyc5M9xKRfywZsKuoZY= github.com/apache/thrift v0.15.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= @@ -995,6 +997,8 @@ github.com/buengese/sgzip v0.1.1 h1:ry+T8l1mlmiWEsDrH/YHZnCVWD2S3im1KLsyO+8ZmTU= github.com/buengese/sgzip v0.1.1/go.mod h1:i5ZiXGF3fhV7gL1xaRRL1nDnmpNj0X061FQzOS8VMas= github.com/bufbuild/protocompile v0.10.0 h1:+jW/wnLMLxaCEG8AX9lD0bQ5v9h1RUiMKOBOT5ll9dM= github.com/bufbuild/protocompile v0.10.0/go.mod h1:G9qQIQo0xZ6Uyj6CMNz0saGmx2so+KONo8/KrELABiY= +github.com/bwmarrin/snowflake v0.3.0 h1:xm67bEhkKh6ij1790JB83OujPR5CzNe8QuQqAgISZN0= +github.com/bwmarrin/snowflake v0.3.0/go.mod h1:NdZxfVWX+oR6y2K0o6qAYv6gIOP9rjG0/E9WsDpxqwE= github.com/caio/go-tdigest v3.1.0+incompatible h1:uoVMJ3Q5lXmVLCCqaMGHLBWnbGoN6Lpu7OAUPR60cds= github.com/caio/go-tdigest v3.1.0+incompatible/go.mod h1:sHQM/ubZStBUmF1WbB8FAm8q9GjDajLC5T7ydxE3JHI= github.com/caio/go-tdigest/v4 v4.0.1 h1:sx4ZxjmIEcLROUPs2j1BGe2WhOtHD6VSe6NNbBdKYh4= @@ -2053,10 +2057,14 @@ github.com/openzipkin/zipkin-go v0.4.3 h1:9EGwpqkgnwdEIJ+Od7QVSEIH+ocmm5nPat0G7s github.com/openzipkin/zipkin-go v0.4.3/go.mod h1:M9wCJZFWCo2RiY+o1eBCEMe0Dp2S5LDHcMZmk3RmK7c= github.com/oracle/oci-go-sdk/v65 v65.69.2 h1:lROMJ8/VakGOGObAWUxTVY2AX1wQCUIzVqfL4Fb2Ay8= github.com/oracle/oci-go-sdk/v65 v65.69.2/go.mod h1:IBEV9l1qBzUpo7zgGaRUhbB05BVfcDGYRFBCPlTcPp0= +github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c h1:rp5dCmg/yLR3mgFuSOe4oEnDDmGLROTvMragMUXpTQw= +github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c/go.mod h1:X07ZCGwUbLaax7L0S3Tw4hpejzu63ZrrQiUe6W0hcy0= github.com/p4lang/p4runtime v1.4.0 h1:LbCCClz/5uJzLU+puL2aA/0Bz6xiZKxKVyVlTIhAWOQ= github.com/p4lang/p4runtime v1.4.0/go.mod h1:OWAP4Wh9uKGnQjleslObpFE0REP78b5gR1pHyYmvNPQ= -github.com/panjf2000/ants/v2 v2.9.1 h1:Q5vh5xohbsZXGcD6hhszzGqB7jSSc2/CRr3QKIga8Kw= -github.com/panjf2000/ants/v2 v2.9.1/go.mod h1:7ZxyxsqE4vvW0M7LSD8aI3cKwgFhBHbxnlN8mDqHa1I= +github.com/panjf2000/ants/v2 v2.10.0 h1:zhRg1pQUtkyRiOFo2Sbqwjp0GfBNo9cUY2/Grpx1p+8= +github.com/panjf2000/ants/v2 v2.10.0/go.mod h1:7ZxyxsqE4vvW0M7LSD8aI3cKwgFhBHbxnlN8mDqHa1I= +github.com/panjf2000/gnet/v2 v2.5.7 h1:EGGIfLYEVAp2l5WSYT2XddSjpQ642PjwphbWhcJ0WBY= +github.com/panjf2000/gnet/v2 v2.5.7/go.mod h1:ppopMJ8VrDbJu8kDsqFQTgNmpMS8Le5CmPxISf+Sauk= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.1 h1:Ah6WQ56rZONR3RW3qWa2NCZ6JAVvSpUcoLBaOmYFt9Q= @@ -2433,6 +2441,8 @@ github.com/zeebo/errs v1.3.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtC github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= +github.com/zentures/cityhash v0.0.0-20131128155616-cdd6a94144ab h1:BD4YbH4Y0ysgbrP9jGuDB0BxkqyTRk6Y70o3D5Z5ayc= +github.com/zentures/cityhash v0.0.0-20131128155616-cdd6a94144ab/go.mod h1:SvJE1nX57VqPOyqkQGEGcJPWZqeB3FCZ8s7a0uSlG+A= github.com/zitadel/logging v0.6.1 h1:Vyzk1rl9Kq9RCevcpX6ujUaTYFX43aa4LkvV1TvUk+Y= github.com/zitadel/logging v0.6.1/go.mod h1:Y4CyAXHpl3Mig6JOszcV5Rqqsojj+3n7y2F591Mp/ow= github.com/zitadel/oidc/v3 v3.30.0 h1:1IuZlK+X+JLExEA2PYgRlVvWHBhz/cMwT7VL/YrQabw= @@ -2551,6 +2561,7 @@ golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1m golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= golang.org/x/crypto v0.20.0/go.mod h1:Xwo95rrVNIoSMx9wa1JroENMToLWn3RNVrTBpLHgZPQ= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -2706,6 +2717,7 @@ golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI= golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -2893,6 +2905,7 @@ golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -2912,6 +2925,7 @@ golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2933,6 +2947,7 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -2941,6 +2956,7 @@ golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ= golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -3346,6 +3362,8 @@ gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/olivere/elastic.v5 v5.0.86 h1:xFy6qRCGAmo5Wjx96srho9BitLhZl2fcnpuidPwduXM= gopkg.in/olivere/elastic.v5 v5.0.86/go.mod h1:M3WNlsF+WhYn7api4D87NIflwTV/c0iVs8cqfWhK+68= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= @@ -3485,4 +3503,4 @@ storj.io/infectious v0.0.2/go.mod h1:QEjKKww28Sjl1x8iDsjBpOM4r1Yp8RsowNcItsZJ1Vs storj.io/picobuf v0.0.3 h1:xAUPB5ZUGfxkqd3bnw3zp01kkWb9wlhg4vtZWUs2S9A= storj.io/picobuf v0.0.3/go.mod h1:4V4xelV1RSCck5GgmkL/Txw9l6IfX3XcBzegmL5Kudo= storj.io/uplink v1.13.1 h1:C8RdW/upALoCyuF16Lod9XGCXEdbJAS+ABQy9JO/0pA= -storj.io/uplink v1.13.1/go.mod h1:x0MQr4UfFsQBwgVWZAtEsLpuwAn6dg7G0Mpne1r516E= +storj.io/uplink v1.13.1/go.mod h1:x0MQr4UfFsQBwgVWZAtEsLpuwAn6dg7G0Mpne1r516E= \ No newline at end of file diff --git a/plugins/outputs/all/inlong.go b/plugins/outputs/all/inlong.go new file mode 100644 index 0000000000000..2b3a360b709e1 --- /dev/null +++ b/plugins/outputs/all/inlong.go @@ -0,0 +1,5 @@ +//go:build !custom || outputs || outputs.inlong + +package all + +import _ "github.com/influxdata/telegraf/plugins/outputs/inlong" // register plugin diff --git a/plugins/outputs/inlong/README.md b/plugins/outputs/inlong/README.md new file mode 100644 index 0000000000000..0467f36c4e4f8 --- /dev/null +++ b/plugins/outputs/inlong/README.md @@ -0,0 +1,44 @@ +# Inlong Output Plugin + +This plugin writes telegraf metrics to +[Apache InLong](https://inlong.apache.org/docs/next/introduction). + +## Global configuration options + +In addition to the plugin-specific configuration settings, plugins support +additional global and plugin configuration settings. These settings are used to +modify metrics, tags, and field or create aliases and configure ordering, etc. +See the [CONFIGURATION.md][CONFIGURATION.md] for more details. + +[CONFIGURATION.md]: ../../../docs/CONFIGURATION.md#plugins + +## Configuration + +```toml @sample.conf +# Send telegraf metrics to Inlong +[[outputs.inlong]] + ## From the Inlong system, data streams group, it contains multiple data streams, and one Group represents + ## one data business unit. + group_id = "test_group" + + ## From the Inlong system, data stream, a stream has a specific data source, data format and data sink. + stream_id = "test_stream" + + ## The URL used to obtain the Inlong DataProxy IP list to which the data will be sent + manager_url = "http://127.0.0.1:8083" + + ## Data format to output. + ## Each data format has its own unique set of configuration options, read + ## more about them here: + ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md + data_format = "csv" + + ## The delimiter used when serializing data in CSV format needs to be consistent with the delimiter + ## configured for Inlong, so that the data can be parsed properly after it reaches Inlong. + ## It can be a space, vertical bar (|), comma (,), semicolon (;), asterisk (*), double quotes ("), etc. + csv_separator = "|" + + ## The final output field order here needs to be consistent with the field order defined by the data + ## stream in Inlong + csv_columns = ["field.key","file.value"] +``` diff --git a/plugins/outputs/inlong/inlong.go b/plugins/outputs/inlong/inlong.go new file mode 100644 index 0000000000000..3817bb9e2d04e --- /dev/null +++ b/plugins/outputs/inlong/inlong.go @@ -0,0 +1,89 @@ +package inlong + +import ( + "context" + _ "embed" + "fmt" + + "github.com/apache/inlong/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/dataproxy" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/internal" + "github.com/influxdata/telegraf/plugins/outputs" +) + +//go:embed sample.conf +var sampleConfig string + +const managerURLSuffix = "/inlong/manager/openapi/dataproxy/getIpList" + +type Inlong struct { + GroupID string `toml:"group_id"` + StreamID string `toml:"stream_id"` + ManagerURL string `toml:"manager_url"` + Log telegraf.Logger `toml:"-"` + + producerFunc func(groupId string, managerUrl string) (dataproxy.Client, error) + producer dataproxy.Client + serializer telegraf.Serializer +} + +func (i *Inlong) SampleConfig() string { + return sampleConfig +} + +func (i *Inlong) SetSerializer(serializer telegraf.Serializer) { + i.serializer = serializer +} + +func (i *Inlong) Connect() error { + producer, err := i.producerFunc(i.GroupID, i.ManagerURL+managerURLSuffix) + if err != nil { + return &internal.StartupError{Err: err, Retry: true} + } + i.producer = producer + return nil +} + +func (i *Inlong) Close() error { + i.producer.Close() + return nil +} + +func (i *Inlong) Write(metrics []telegraf.Metric) error { + for _, metric := range metrics { + b, err := i.serializer.Serialize(metric) + if err != nil { + return fmt.Errorf("could not serialize metric: %w", err) + } + err = i.producer.Send(context.Background(), dataproxy.Message{ + GroupID: i.GroupID, + StreamID: i.StreamID, + Payload: b, + }) + if err != nil { + return err + } + } + return nil +} + +func init() { + outputs.Add("inlong", func() telegraf.Output { + return &Inlong{ + producerFunc: newProducer, + } + }) +} + +func newProducer(groupID, managerURL string) (dataproxy.Client, error) { + producer, err := dataproxy.NewClient( + dataproxy.WithGroupID(groupID), + dataproxy.WithURL(managerURL), + ) + if err != nil { + fmt.Println(err) + return nil, err + } + return producer, nil +} diff --git a/plugins/outputs/inlong/inlong_test.go b/plugins/outputs/inlong/inlong_test.go new file mode 100644 index 0000000000000..be0bef2462c70 --- /dev/null +++ b/plugins/outputs/inlong/inlong_test.go @@ -0,0 +1,68 @@ +package inlong + +import ( + "context" + "testing" + "time" + + "github.com/apache/inlong/inlong-sdk/dataproxy-sdk-twins/dataproxy-sdk-golang/dataproxy" + "github.com/stretchr/testify/require" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/metric" + "github.com/influxdata/telegraf/plugins/serializers/csv" +) + +type MockProducer struct { + groupID string + managerURL string +} + +func (p *MockProducer) Send(context.Context, dataproxy.Message) error { + return nil +} + +func (p *MockProducer) SendAsync(context.Context, dataproxy.Message, dataproxy.Callback) { +} + +func (p *MockProducer) Close() { +} + +func (p *MockProducer) SendMessage(context.Context, dataproxy.Message) error { + return nil +} + +func NewMockProducer(groupID, managerURL string) (dataproxy.Client, error) { + p := &MockProducer{} + p.groupID = groupID + p.managerURL = managerURL + return p, nil +} + +func TestInlong_Connect(t *testing.T) { + i := &Inlong{producerFunc: NewMockProducer} + require.NoError(t, i.Connect()) +} + +func TestInlong_Write(t *testing.T) { + s := &csv.Serializer{Header: true} + require.NoError(t, s.Init()) + producer := &MockProducer{} + i := &Inlong{ + producer: producer, + serializer: s, + } + m := metric.New( + "cpu", + map[string]string{ + "topic": "test-topic", + }, + map[string]interface{}{ + "value": 42.0, + }, + time.Unix(0, 0), + ) + var metrics []telegraf.Metric + metrics = append(metrics, m) + require.NoError(t, i.Write(metrics)) +} diff --git a/plugins/outputs/inlong/sample.conf b/plugins/outputs/inlong/sample.conf new file mode 100644 index 0000000000000..e32fd1a07b048 --- /dev/null +++ b/plugins/outputs/inlong/sample.conf @@ -0,0 +1,27 @@ +# Send telegraf metrics to Inlong +[[outputs.inlong]] + ## From the Inlong system, data streams group, it contains multiple data streams, and one Group represents + ## one data business unit. + group_id = "test_group" + + ## From the Inlong system, data stream, a stream has a specific data source, data format and data sink. + stream_id = "test_stream" + + ## The URL used to obtain the Inlong DataProxy IP list to which the data will be sent + manager_url = "http://127.0.0.1:8083" + + ## Data format to output. + ## Each data format has its own unique set of configuration options, read + ## more about them here: + ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md + ## Suggest using CSV format here, as Inlong is also processed in CSV format + data_format = "csv" + + ## The delimiter used when serializing data in CSV format needs to be consistent with the delimiter + ## configured for Inlong, so that the data can be parsed properly after it reaches Inlong. + ## It can be a space, vertical bar (|), comma (,), semicolon (;), asterisk (*), double quotes ("), etc. + csv_separator = "|" + + ## The final output field order here needs to be consistent with the field order defined by the data + ## stream in Inlong + csv_columns = ["field.key","file.value"]