Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

BUG: diffrent request /api/v1/query response same result #587

Open
wu0407 opened this issue Nov 3, 2021 · 1 comment
Open

BUG: diffrent request /api/v1/query response same result #587

wu0407 opened this issue Nov 3, 2021 · 1 comment

Comments

@wu0407
Copy link
Contributor

wu0407 commented Nov 3, 2021 •
edited
Loading

trickster version: v1.1.5
Use trickster in frontend of VictoriaMetrics, set vmalert datasource use trickster. when vmalert send sequence request diffrent /api/v1/query in one tcp connection, trickster response same cache reult.

                                                                                                
+---------------------+             +---------------------+              +---------------------+
|                     |             |                     |              |                     |
|       vmalert       |--------------       trickster     ----------------         vmselect    |
|                     |             |                     |              |                     |
+---------------------+             +---------------------+              +---------------------+

trickster configure:

  trickster.conf: |-
    [main]

    [frontend]
    listen_port = 8480
    connections_limit = 0

    [caches]

      [caches.default]
      cache_type = "redis"

        [caches.default.redis]
        client_type = "cluster"
        password = "bFdfkKWSbT"
        endpoints = [ 'redis-cluster-0.redis-cluster-headless.victoria:6379', 'redis-cluster-1.redis-cluster-headless.victoria:6379', 'redis-cluster-2.redis-cluster-headless.victoria:6379' ]

    [origins]

      [origins.default]
      origin_type = "prometheus"
      origin_url = "http://vmselect-victoria-metrics-k8s-stack.victoria.svc:8481/select/0/prometheus"
      health_check_upstream_path = "/select/0/prometheus/api/v1/query"

    [metrics]
    listen_port = 8481

    [logging]
    log_level = "debug"

trickster configure dump:

[main]
  instance_id = 0
  config_handler_path = "/trickster/config"
  ping_handler_path = "/trickster/ping"
  reload_handler_path = "/trickster/config/reload"
  health_handler_path = "/trickster/health"
  pprof_server = "both"
  server_name = "trickster-6d669ccbfc-bv7n2"

[origins]
  [origins.default]
    origin_type = "prometheus"
    origin_url = "http://vmselect-victoria-metrics-k8s-stack.victoria.svc:8481/select/0/prometheus"
    timeout_secs = 180
    keep_alive_timeout_secs = 300
    max_idle_conns = 20
    cache_name = "default"
    cache_key_prefix = "vmselect-victoria-metrics-k8s-stack.victoria.svc:8481"
    health_check_upstream_path = "/select/0/prometheus/api/v1/query"
    health_check_verb = "GET"
    health_check_query = "query=up"
    timeseries_retention_factor = 1024
    timeseries_eviction_method = "oldest"
    backfill_tolerance_secs = 0
    negative_cache_name = "default"
    timeseries_ttl_secs = 21600
    fastforward_ttl_secs = 15
    max_ttl_secs = 86400
    revalidation_factor = 2.0
    max_object_size_bytes = 524288
    compressable_types = ["text/html", "text/javascript", "text/css", "text/plain", "text/xml", "text/json", "application/json", "application/javascript", "application/xml"]
    tracing_name = "default"
    rule_name = ""
    req_rewriter_name = ""
    forwarded_headers = "standard"
    is_default = true
    fast_forward_disable = false
    path_routing_disabled = false
    require_tls = false
    multipart_ranges_disabled = false
    dearticulate_upstream_ranges = false
    [origins.default.health_check_headers]
    [origins.default.paths]
      [origins.default.paths."/-GET-POST"]
        path = "/"
        match_type = "prefix"
        handler = "proxy"
        methods = ["GET", "POST"]
        cache_key_params = []
        cache_key_headers = []
        cache_key_form_fields = []
        response_code = 0
        response_body = ""
        collapsed_forwarding = ""
        req_rewriter_name = ""
        no_metrics = false
      [origins.default.paths."/api/v1/-GET-POST"]
        path = "/api/v1/"
        match_type = "prefix"
        handler = "proxy"
        methods = ["GET", "POST"]
        cache_key_params = []
        cache_key_headers = []
        cache_key_form_fields = []
        response_code = 0
        response_body = ""
        collapsed_forwarding = ""
        req_rewriter_name = ""
        no_metrics = false
      [origins.default.paths."/api/v1/alertmanagers-GET"]
        path = "/api/v1/alertmanagers"
        match_type = "exact"
        handler = "proxycache"
        methods = ["GET"]
        cache_key_params = []
        cache_key_headers = []
        cache_key_form_fields = []
        response_code = 0
        response_body = ""
        collapsed_forwarding = ""
        req_rewriter_name = ""
        no_metrics = false
        [origins.default.paths."/api/v1/alertmanagers-GET".response_headers]
          Cache-Control = "s-maxage=30"
      [origins.default.paths."/api/v1/alerts-GET"]
        path = "/api/v1/alerts"
        match_type = "exact"
        handler = "proxycache"
        methods = ["GET"]
        cache_key_params = []
        cache_key_headers = []
        cache_key_form_fields = []
        response_code = 0
        response_body = ""
        collapsed_forwarding = ""
        req_rewriter_name = ""
        no_metrics = false
        [origins.default.paths."/api/v1/alerts-GET".response_headers]
          Cache-Control = "s-maxage=30"
      [origins.default.paths."/api/v1/label/-GET"]
        path = "/api/v1/label/"
        match_type = "prefix"
        handler = "proxycache"
        methods = ["GET"]
        cache_key_params = []
        cache_key_headers = []
        cache_key_form_fields = []
        response_code = 0
        response_body = ""
        collapsed_forwarding = ""
        req_rewriter_name = ""
        no_metrics = false
        [origins.default.paths."/api/v1/label/-GET".response_headers]
          Cache-Control = "s-maxage=30"
      [origins.default.paths."/api/v1/labels-GET-POST"]
        path = "/api/v1/labels"
        match_type = "exact"
        handler = "proxycache"
        methods = ["GET", "POST"]
        cache_key_params = []
        cache_key_headers = []
        cache_key_form_fields = []
        response_code = 0
        response_body = ""
        collapsed_forwarding = ""
        req_rewriter_name = ""
        no_metrics = false
        [origins.default.paths."/api/v1/labels-GET-POST".response_headers]
          Cache-Control = "s-maxage=30"
      [origins.default.paths."/api/v1/query-GET-POST"]
        path = "/api/v1/query"
        match_type = "exact"
        handler = "query"
        methods = ["GET", "POST"]
        cache_key_params = ["query", "time"]
        cache_key_headers = []
        cache_key_form_fields = []
        response_code = 0
        response_body = ""
        collapsed_forwarding = ""
        req_rewriter_name = ""
        no_metrics = false
        [origins.default.paths."/api/v1/query-GET-POST".response_headers]
          Cache-Control = "s-maxage=30"
      [origins.default.paths."/api/v1/query_range-GET-POST"]
        path = "/api/v1/query_range"
        match_type = "exact"
        handler = "query_range"
        methods = ["GET", "POST"]
        cache_key_params = ["query", "step"]
        cache_key_headers = []
        cache_key_form_fields = []
        response_code = 0
        response_body = ""
        collapsed_forwarding = ""
        req_rewriter_name = ""
        no_metrics = false
        [origins.default.paths."/api/v1/query_range-GET-POST".response_headers]
          Cache-Control = "s-maxage=21600"
      [origins.default.paths."/api/v1/rules-GET"]
        path = "/api/v1/rules"
        match_type = "exact"
        handler = "proxycache"
        methods = ["GET"]
        cache_key_params = []
        cache_key_headers = []
        cache_key_form_fields = []
        response_code = 0
        response_body = ""
        collapsed_forwarding = ""
        req_rewriter_name = ""
        no_metrics = false
        [origins.default.paths."/api/v1/rules-GET".response_headers]
          Cache-Control = "s-maxage=30"
      [origins.default.paths."/api/v1/series-GET-POST"]
        path = "/api/v1/series"
        match_type = "exact"
        handler = "series"
        methods = ["GET", "POST"]
        cache_key_params = ["match[]", "start", "end"]
        cache_key_headers = []
        cache_key_form_fields = []
        response_code = 0
        response_body = ""
        collapsed_forwarding = ""
        req_rewriter_name = ""
        no_metrics = false
        [origins.default.paths."/api/v1/series-GET-POST".response_headers]
          Cache-Control = "s-maxage=30"
      [origins.default.paths."/api/v1/status-GET"]
        path = "/api/v1/status"
        match_type = "prefix"
        handler = "proxycache"
        methods = ["GET"]
        cache_key_params = []
        cache_key_headers = []
        cache_key_form_fields = []
        response_code = 0
        response_body = ""
        collapsed_forwarding = ""
        req_rewriter_name = ""
        no_metrics = false
        [origins.default.paths."/api/v1/status-GET".response_headers]
          Cache-Control = "s-maxage=30"
      [origins.default.paths."/api/v1/targets-GET"]
        path = "/api/v1/targets"
        match_type = "exact"
        handler = "proxycache"
        methods = ["GET"]
        cache_key_params = []
        cache_key_headers = []
        cache_key_form_fields = []
        response_code = 0
        response_body = ""
        collapsed_forwarding = ""
        req_rewriter_name = ""
        no_metrics = false
        [origins.default.paths."/api/v1/targets-GET".response_headers]
          Cache-Control = "s-maxage=30"
      [origins.default.paths."/api/v1/targets/metadata-GET"]
        path = "/api/v1/targets/metadata"
        match_type = "exact"
        handler = "proxycache"
        methods = ["GET"]
        cache_key_params = ["match_target", "metric", "limit"]
        cache_key_headers = []
        cache_key_form_fields = []
        response_code = 0
        response_body = ""
        collapsed_forwarding = ""
        req_rewriter_name = ""
        no_metrics = false
        [origins.default.paths."/api/v1/targets/metadata-GET".response_headers]
          Cache-Control = "s-maxage=30"
    [origins.default.tls]
      full_chain_cert_path = ""
      private_key_path = ""
      insecure_skip_verify = false
      client_cert_path = ""
      client_key_path = ""

[caches]
  [caches.default]
    cache_type = "redis"
    [caches.default.index]
      reap_interval_secs = 3
      flush_interval_secs = 5
      max_size_bytes = 536870912
      max_size_backoff_bytes = 16777216
      max_size_objects = 0
      max_size_backoff_objects = 100
    [caches.default.redis]
      client_type = "cluster"
      protocol = "tcp"
      endpoint = "redis:6379"
      endpoints = ["redis-cluster-0.redis-cluster-headless.victoria:6379", "redis-cluster-1.redis-cluster-headless.victoria:6379", "redis-cluster-2.redis-cluster-headless.victoria:6379"]
      password = "*****"
      sentinel_master = ""
      db = 0
      max_retries = 0
      min_retry_backoff_ms = 0
      max_retry_backoff_ms = 0
      dial_timeout_ms = 0
      read_timeout_ms = 0
      write_timeout_ms = 0
      pool_size = 0
      min_idle_conns = 0
      max_conn_age_ms = 0
      pool_timeout_ms = 0
      idle_timeout_ms = 0
      idle_check_frequency_ms = 0
    [caches.default.filesystem]
      cache_path = "/tmp/trickster"
    [caches.default.bbolt]
      filename = "trickster.db"
      bucket = "trickster"
    [caches.default.badger]
      directory = "/tmp/trickster"
      value_directory = "/tmp/trickster"

[frontend]
  listen_address = ""
  listen_port = 8480
  tls_listen_address = ""
  tls_listen_port = 8483
  connections_limit = 0

[logging]
  log_file = ""
  log_level = "debug"

[metrics]
  listen_address = ""
  listen_port = 8481

[tracing]
  [tracing.default]
    tracer_type = "none"
    service_name = "trickster"
    collector_url = ""
    collector_user = ""
    collector_pass = ""
    sample_rate = 1.0
    [tracing.default.stdout]
      pretty_print = false
    [tracing.default.jaeger]
      endpoint_type = ""

[negative_caches]
  [negative_caches.default]

[reloading]
  listen_address = "127.0.0.1"
  listen_port = 8484
  handler_path = "/trickster/config/reload"
  drain_timeout_secs = 30
  rate_limit_secs = 3

tcpdump http flow:

POST /api/v1/query?query=kube_hpa_status_condition%7Bcondition%3D%22AbleToScale%22%2C+status%3D%22false%22%7D+%2A+on%28hpa%2Cnamespace%29++group_left%28label_app%29+kube_hpa_labels+%3D%3D+1&step=15s&time=1635906330 HTTP/1.1
Host: trickster.victoria.svc:8480
User-Agent: Go-http-client/1.1
Content-Length: 0
Content-Type: application/json; charset=utf-8
Accept-Encoding: gzip

HTTP/1.1 200 OK
Access-Control-Allow-Origin: *
Cache-Control: s-maxage=30
Content-Encoding: gzip
Content-Type: application/json; charset=utf-8
Vary: Accept-Encoding
X-Server-Hostname: vmselect-victoria-metrics-k8s-stack-2
X-Trickster-Result: engine=ObjectProxyCache; status=hit
Date: Wed, 03 Nov 2021 02:25:37 GMT
Content-Length: 346

{"status":"success","isPartial":false,"data":{"resultType":"vector","result":[{"metric":{"backend":"bizmap-backend.internal.xxx.cn/","cluster":"k8s-online-victoria"},"value":[1635906315,"4168.1701488"]},{"metric":{"backend":"cdp.collect.pid.k8s1.internal.xxx.com/","cluster":"k8s-online-victoria"},"value":[1635906315,"325.85644152"]},{"metric":{"backend":"cdp.collect.pid.k8s1.internal.xxx.com/","cluster":"k8scs1-online-sh5-victoria"},"value":[1635906315,"282.73285584"]},{"metric":{"backend":"dopen.xxx.com/","cluster":"k8s-online-victoria"},"value":[1635906315,"21234.966215028002"]},{"metric":{"backend":"dopen.xxx.com/fuwu/b/","cluster":"k8s-online-victoria"},"value":[1635906315,"1774.2817677599999"]},{"metric":{"backend":"jscracker.internal.xxx.cn/","cluster":"k8s-online-victoria"},"value":[1635906315,"828222.4420423079"]},{"metric":{"backend":"saas-fe-zeus-common.internal.xxx.com/","cluster":"k8s-online-victoria"},"value":[1635906315,"129.67017696"]}]}}POST /api/v1/query?query=absent%28up%7Bjob%3D%22kube-controller-manager%22%7D+%3D%3D+1%29&step=15s&time=1635906330 HTTP/1.1
Host: trickster.victoria.svc:8480
User-Agent: Go-http-client/1.1
Content-Length: 0
Content-Type: application/json; charset=utf-8
Accept-Encoding: gzip

HTTP/1.1 200 OK
Access-Control-Allow-Origin: *
Cache-Control: s-maxage=30
Content-Encoding: gzip
Content-Type: application/json; charset=utf-8
Vary: Accept-Encoding
X-Server-Hostname: vmselect-victoria-metrics-k8s-stack-2
X-Trickster-Result: engine=ObjectProxyCache; status=hit
Date: Wed, 03 Nov 2021 02:25:37 GMT
Content-Length: 346

{"status":"success","isPartial":false,"data":{"resultType":"vector","result":[{"metric":{"backend":"bizmap-backend.internal.xxx.cn/","cluster":"k8s-online-victoria"},"value":[1635906315,"4168.1701488"]},{"metric":{"backend":"cdp.collect.pid.k8s1.internal.xxx.com/","cluster":"k8s-online-victoria"},"value":[1635906315,"325.85644152"]},{"metric":{"backend":"cdp.collect.pid.k8s1.internal.xxx.com/","cluster":"k8scs1-online-sh5-victoria"},"value":[1635906315,"282.73285584"]},{"metric":{"backend":"dopen.xxx.com/","cluster":"k8s-online-victoria"},"value":[1635906315,"21234.966215028002"]},{"metric":{"backend":"dopen.xxx.com/fuwu/b/","cluster":"k8s-online-victoria"},"value":[1635906315,"1774.2817677599999"]},{"metric":{"backend":"jscracker.internal.xxx.cn/","cluster":"k8s-online-victoria"},"value":[1635906315,"828222.4420423079"]},{"metric":{"backend":"saas-fe-zeus-common.internal.xxx.com/","cluster":"k8s-online-victoria"},"value":[1635906315,"129.67017696"]}]}}
@wu0407 wu0407 changed the title sequence request diffrent /api/v1/query in one tcp connection, response same result BUG: diffrent request /api/v1/query response same result Nov 8, 2021
@wu0407
Copy link
Contributor Author

wu0407 commented Nov 8, 2021

Test in v2.0.0-beta2 same result

@wu0407 wu0407 mentioned this issue Nov 8, 2021
Merged
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees
No one assigned
Labels
None yet
Projects
None yet
Milestone
No milestone
Development

No branches or pull requests
1 participant
@wu0407