[BUG] target missing in response

[zivillian]

Describe the bug
A request with multiple similar targets is not returning all results if they use multiple metrics and one of them does not exist.

CarbonAPI Version
0.16.0 and 0.17.0

Logs

will try to fetch data        {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b"}
single fetch        {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "client": {}}
waiting for slot        {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "localhost", "max_connections": 0}
got slot        {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "localhost"}
sending request        {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "localhost"}
will try to fetch data        {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b"}
single fetch        {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "client": {}}
waiting for slot        {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "http://127.0.0.1:8080", "max_connections": 0}
got slot        {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "http://127.0.0.1:8080"}
sending request        {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "http://127.0.0.1:8080"}
trying to get slot        {"type": "protoV3Group", "name": "http://127.0.0.1:8080", "type": "fetch", "request": "&MultiFetchRequest{Metrics:[]FetchRequest{FetchRequest{Name:carbon.api.host01.requests,StartTime:1727733600,StopTime:1730318365,HighPrecisionTimestamps:false,PathExpression:carbon.api.host01.requests,FilterFunctions:[]*FilteringFunction{},MaxDataPoints:1306,},FetchRequest{Name:carbon.api.host02.requests,StartTime:1727733600,StopTime:1730318365,HighPrecisionTimestamps:false,PathExpression:carbon.api.host02.requests,FilterFunctions:[]*FilteringFunction{},MaxDataPoints:1306,},},}", "function": "HttpQuery.doRequest", "server": "http://127.0.0.1:8080", "name": "http://127.0.0.1:8080", "uri": "http://127.0.0.1:8080/render/?format=carbonapi_v3_pb", "name": "http://127.0.0.1:8080"}
got slot for server        {"type": "protoV3Group", "name": "http://127.0.0.1:8080", "type": "fetch", "request": "&MultiFetchRequest{Metrics:[]FetchRequest{FetchRequest{Name:carbon.api.host01.requests,StartTime:1727733600,StopTime:1730318365,HighPrecisionTimestamps:false,PathExpression:carbon.api.host01.requests,FilterFunctions:[]*FilteringFunction{},MaxDataPoints:1306,},FetchRequest{Name:carbon.api.host02.requests,StartTime:1727733600,StopTime:1730318365,HighPrecisionTimestamps:false,PathExpression:carbon.api.host02.requests,FilterFunctions:[]*FilteringFunction{},MaxDataPoints:1306,},},}", "function": "HttpQuery.doRequest", "server": "http://127.0.0.1:8080", "name": "http://127.0.0.1:8080", "uri": "http://127.0.0.1:8080/render/?format=carbonapi_v3_pb", "name": "http://127.0.0.1:8080"}
got response        {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "http://127.0.0.1:8080", "metrics_in_response": 1, "errors_count": 0, "timeouts_count": 0, "render_requests_count": 1, "render_errors_count": 0, "render_timeouts_count": 0, "zipper_requests_count": 0, "total_metric_count": 0, "servers_count": 0, "failed_servers_count": 0}
got response (after merge)        {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "http://127.0.0.1:8080", "metrics_in_response": 1, "errors_count": 0, "timeouts_count": 0, "render_requests_count": 1, "render_errors_count": 0, "render_timeouts_count": 0, "zipper_requests_count": 0, "total_metric_count": 0, "servers_count": 0, "failed_servers_count": 0}
got some fetch responses        {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "backends_count": 1, "response_count": 1, "have_errors": false, "errors": null, "metrics_in_response": 1}
got response        {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "localhost", "metrics_in_response": 1, "errors_count": 0, "timeouts_count": 0, "render_requests_count": 1, "render_errors_count": 0, "render_timeouts_count": 0, "zipper_requests_count": 0, "total_metric_count": 0, "servers_count": 0, "failed_servers_count": 0}
got response (after merge)        {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "localhost", "metrics_in_response": 1, "errors_count": 0, "timeouts_count": 0, "render_requests_count": 1, "render_errors_count": 0, "render_timeouts_count": 0, "zipper_requests_count": 0, "total_metric_count": 0, "servers_count": 0, "failed_servers_count": 0}
got some fetch responses        {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host01.requests", "carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "backends_count": 1, "response_count": 1, "have_errors": false, "errors": null, "metrics_in_response": 1}
will try to fetch data        {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b"}
single fetch        {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "client": {}}
waiting for slot        {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "localhost", "max_connections": 0}
got slot        {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "localhost"}
sending request        {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "localhost"}
will try to fetch data        {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b"}
single fetch        {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "client": {}}
waiting for slot        {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "http://127.0.0.1:8080", "max_connections": 0}
got slot        {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "http://127.0.0.1:8080"}
sending request        {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "http://127.0.0.1:8080"}
trying to get slot        {"type": "protoV3Group", "name": "http://127.0.0.1:8080", "type": "fetch", "request": "&MultiFetchRequest{Metrics:[]FetchRequest{FetchRequest{Name:carbon.api.host02.requests,StartTime:1727733600,StopTime:1730318365,HighPrecisionTimestamps:false,PathExpression:carbon.api.host02.requests,FilterFunctions:[]*FilteringFunction{},MaxDataPoints:1306,},},}", "function": "HttpQuery.doRequest", "server": "http://127.0.0.1:8080", "name": "http://127.0.0.1:8080", "uri": "http://127.0.0.1:8080/render/?format=carbonapi_v3_pb", "name": "http://127.0.0.1:8080"}
got slot for server        {"type": "protoV3Group", "name": "http://127.0.0.1:8080", "type": "fetch", "request": "&MultiFetchRequest{Metrics:[]FetchRequest{FetchRequest{Name:carbon.api.host02.requests,StartTime:1727733600,StopTime:1730318365,HighPrecisionTimestamps:false,PathExpression:carbon.api.host02.requests,FilterFunctions:[]*FilteringFunction{},MaxDataPoints:1306,},},}", "function": "HttpQuery.doRequest", "server": "http://127.0.0.1:8080", "name": "http://127.0.0.1:8080", "uri": "http://127.0.0.1:8080/render/?format=carbonapi_v3_pb", "name": "http://127.0.0.1:8080"}
got response        {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "http://127.0.0.1:8080", "metrics_in_response": 0, "errors_count": 0, "timeouts_count": 0, "render_requests_count": 1, "render_errors_count": 0, "render_timeouts_count": 0, "zipper_requests_count": 0, "total_metric_count": 0, "servers_count": 0, "failed_servers_count": 0}
got response (after merge)        {"type": "broadcastGroup", "groupName": "localhost", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "http://127.0.0.1:8080", "metrics_in_response": 0, "errors_count": 0, "timeouts_count": 0, "render_requests_count": 1, "render_errors_count": 0, "render_timeouts_count": 0, "zipper_requests_count": 0, "total_metric_count": 0, "servers_count": 0, "failed_servers_count": 0}
got response        {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "localhost", "response_is_nil": true, "stats_is_nil": false, "err": "metric not found", "errVerbose": "metric not found\nHTTP Code: 404\n\ngithub.com/go-graphite/carbonapi/zipper/types.init\n\t/root/go/src/github.com/go-graphite/carbonapi/zipper/types/errors.go:22\nruntime.doInit1\n\t/root/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.22.1.linux-amd64/src/runtime/proc.go:7176\nruntime.doInit\n\t/root/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.22.1.linux-amd64/src/runtime/proc.go:7143\nruntime.main\n\t/root/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.22.1.linux-amd64/src/runtime/proc.go:253\nruntime.goexit\n\t/root/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.22.1.linux-amd64/src/runtime/asm_amd64.s:1695"}
got response (after merge)        {"type": "broadcastGroup", "groupName": "root", "type": "fetch", "request": ["carbon.api.host02.requests"], "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "multi_fetch": false, "backend_name": "localhost", "metrics_in_response": 0, "errors_count": 1, "timeouts_count": 0, "render_requests_count": 1, "render_errors_count": 0, "render_timeouts_count": 0, "zipper_requests_count": 0, "total_metric_count": 0, "servers_count": 0, "failed_servers_count": 0}
had errors while fetching result        {"function": "FetchProtoV3", "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "errors": "metric not found", "errorsVerbose": "metric not found\nHTTP Code: 404\n\ngithub.com/go-graphite/carbonapi/zipper/types.init\n\t/root/go/src/github.com/go-graphite/carbonapi/zipper/types/errors.go:22\nruntime.doInit1\n\t/root/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.22.1.linux-amd64/src/runtime/proc.go:7176\nruntime.doInit\n\t/root/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.22.1.linux-amd64/src/runtime/proc.go:7143\nruntime.main\n\t/root/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.22.1.linux-amd64/src/runtime/proc.go:253\nruntime.goexit\n\t/root/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.22.1.linux-amd64/src/runtime/asm_amd64.s:1695", "httpCode": 404}
no metrics fetched        {"function": "FetchProtoV3", "carbonapi_uuid": "de03d1ee-6381-404c-92d9-0c95ce04c74b", "errors": "metric not found", "errorsVerbose": "metric not found\nHTTP Code: 404\n\ngithub.com/go-graphite/carbonapi/zipper/types.init\n\t/root/go/src/github.com/go-graphite/carbonapi/zipper/types/errors.go:22\nruntime.doInit1\n\t/root/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.22.1.linux-amd64/src/runtime/proc.go:7176\nruntime.doInit\n\t/root/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.22.1.linux-amd64/src/runtime/proc.go:7143\nruntime.main\n\t/root/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.22.1.linux-amd64/src/runtime/proc.go:253\nruntime.goexit\n\t/root/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.22.1.linux-amd64/src/runtime/asm_amd64.s:1695"}
request served        {"data": {"handler":"render","carbonapi_uuid":"de03d1ee-6381-404c-92d9-0c95ce04c74b","url":"/render","peer_ip":"198.51.100.24","host":"198.51.100.25:8081","format":"json","use_cache":true,"targets":["alias(group(carbon.api.host01.requests,carbon.api.host02.requests),'1')","alias(group(carbon.api.host01.requests,carbon.api.host02.requests),'3')"],"cache_timeout":60,"metrics":["alias(group(carbon.api.host01.requests,carbon.api.host02.requests),'1')","alias(group(carbon.api.host01.requests,carbon.api.host02.requests),'3')"],"have_non_fatal_errors":true,"runtime":0.015166965,"http_code":200,"carbonzipper_response_size_bytes":344717,"carbonapi_response_size_bytes":22261,"from":1727733600,"until":1730318365,"max_data_points":1306,"from_raw":"1727733600","until_raw":"1730318365","uri":"/render","from_cache":false,"used_backend_cache":false,"request_headers":{}}}

CarbonAPI Configuration:

listen: "[::]:8081"

concurency: 1000
cache:
   type: "null"
   size_mb: 0
   defaultTimeoutSec: 60

cpus: 0
tz: "Europe/Berlin,0"

maxBatchSize: 0
graphite:
    host: "localhost:2003"
    interval: "60s"
    prefix: "carbon.api"
    pattern: "{prefix}.{fqdn}"
idleConnections: 10
pidFile: ""
# See https://github.com/go-graphite/carbonzipper/blob/master/example.conf#L70-L108 for format explanation
upstreams:
    tldCacheDisabled: true
    buckets: 10
    slowLogThreshold: "60s"

    timeouts:
        find: "2s"
        render: "300s"
        connect: "200ms"

    concurrencyLimitPerServer: 0
    keepAliveInterval: "30s"
    maxIdleConnsPerHost: 100
    doMultipleRequestsIfSplit: false

    backendsv2:
        backends:
          -
            groupName: "localhost"
            protocol: "carbonapi_v3_pb"
            lbMethod: "broadcast"
            maxTries: 3
            maxBatchSize: 0
            keepAliveInterval: "10s"
            concurrencyLimit: 0
            maxIdleConnsPerHost: 1000
            forceAttemptHTTP2: false
            doMultipleRequestsIfSplit: false
            servers:
                - "http://127.0.0.1:8080"

expvar:
  enabled: true
  pprofEnabled: true
  listen: "[::]:8082"

logger:
    - logger: ""
      file: "stderr"
      level: "debug"
      encoding: "console"
      encodingTime: "iso8601"
      encodingDuration: "seconds"

Simplified query (if applicable)

?target=alias(group(carbon.api.host01.requests,carbon.api.host02.requests),'1')
&target=alias(group(carbon.api.host01.requests,carbon.api.host02.requests),'3')
&from=1727733600
&until=1730318937
&format=json
&maxDataPoints=1306

In this query we have have two targets which only differ in the alias. The first metric carbon.api.host01.requests does exist, but the second carbon.api.host02.requests does not exist.

The response only contains the result for the first target. In our real example query we have three such targets and only get the result for the first and second.

[
  {
    "target":"1",
    "datapoints":[[156,1727733660],[...],[80.28571428571429,1730317560]],
    "tags":{"name":"carbon.api.host01.requests"}
  }
]