fix(basic-metrics): Remove label from basic dns and filter out IPs from linux util plugins (#877)

# Description

This PR addresses the cardinality issue in two basic plugins. By
removing the label from basic dns and excluding localhost and node ip
from linux util plugin tcp_connection_remote.

It also adds a check in the e2e to validate metrics without checking
specific labels.

## Related Issue

https://github.com/azure-networking/retina-enterprise/issues/263
https://github.com/azure-networking/retina-enterprise/issues/272

## Checklist

- [x] I have read the [contributing
documentation](https://retina.sh/docs/contributing).
- [x] I signed and signed-off the commits (`git commit -S -s ...`). See
[this
documentation](https://docs.github.com/en/authentication/managing-commit-signature-verification/about-commit-signature-verification)
on signing commits.
- [x] I have correctly attributed the author(s) of the code.
- [x] I have tested the changes locally.
- [x] I have followed the project's style guidelines.
- [x] I have updated the documentation, if necessary.
- [ ] I have added tests, if applicable.

## Screenshots (if applicable) or Testing Completed

Please add any relevant screenshots or GIFs to showcase the changes
made.


```
# TYPE networkobservability_dns_request_count counter
networkobservability_dns_request_count 80
# HELP networkobservability_dns_response_count DNS responses by statistics
# TYPE networkobservability_dns_response_count counter
networkobservability_dns_response_count 50

# TYPE networkobservability_adv_dns_request_count counter
networkobservability_adv_dns_request_count{ip="10.224.3.111",namespace="kube-system-test",podname="agnhost-adv-dns-port-forward-7215873926146132341-0",query="some.non.existent.domain.",query_type="A",workload_kind="StatefulSet",workload_name="agnhost-adv-dns-port-forward-7215873926146132341"} 30

# TYPE networkobservability_adv_dns_response_count counter
networkobservability_adv_dns_response_count{ip="10.224.3.111",namespace="kube-system-test",num_response="0",podname="agnhost-adv-dns-port-forward-7215873926146132341-0",query="some.non.existent.domain.",query_type="A",response="",return_code="NXDOMAIN",workload_kind="StatefulSet",workload_name="agnhost-adv-dns-port-forward-7215873926146132341"} 19
```
Before, nodeIP is  10.224.2.241 

```
# TYPE networkobservability_tcp_connection_remote gauge
# TYPE networkobservability_tcp_connection_remote gauge
networkobservability_tcp_connection_remote{address="0.0.0.0",port="0"} 11
**networkobservability_tcp_connection_remote{address="10.224.2.241",port="29613"} 0**
networkobservability_tcp_connection_remote{address="10.224.2.255",port="9090"} 5
networkobservability_tcp_connection_remote{address="10.224.3.2",port="9090"} 1
networkobservability_tcp_connection_remote{address="10.224.3.206",port="8082"} 1
networkobservability_tcp_connection_remote{address="10.224.3.62",port="9090"} 4
networkobservability_tcp_connection_remote{address="104.208.181.172",port="443"} 4
**networkobservability_tcp_connection_remote{address="127.0.0.1",port="50678"} 0
networkobservability_tcp_connection_remote{address="127.0.0.1",port="50684"} 0
networkobservability_tcp_connection_remote{address="127.0.0.1",port="50694"} 0
networkobservability_tcp_connection_remote{address="127.0.0.1",port="55960"} 0**
networkobservability_tcp_connection_remote{address="13.107.246.41",port="443"} 0
networkobservability_tcp_connection_remote{address="150.171.69.10",port="443"} 0
networkobservability_tcp_connection_remote{address="168.63.129.16",port="80"} 0
networkobservability_tcp_connection_remote{address="169.254.169.254",port="80"} 68
networkobservability_tcp_connection_remote{address="20.190.151.68",port="443"} 0
networkobservability_tcp_connection_remote{address="20.209.154.1",port="443"} 0
networkobservability_tcp_connection_remote{address="20.209.155.1",port="443"} 0
networkobservability_tcp_connection_remote{address="20.209.179.65",port="443"} 5
networkobservability_tcp_connection_remote{address="20.36.150.0",port="443"} 0
networkobservability_tcp_connection_remote{address="4.150.240.10",port="443"} 0
networkobservability_tcp_connection_remote{address="40.64.135.140",port="443"} 0
networkobservability_tcp_connection_remote{address="52.179.73.37",port="443"} 0
networkobservability_tcp_connection_remote{address="52.179.73.39",port="443"} 0
networkobservability_tcp_connection_remote{address="52.188.247.147",port="443"} 0
networkobservability_tcp_connection_remote{address="52.253.71.198",port="443"} 4
# HELP networkobservability_tcp_connection_stats TCP connections statistics
```
after. No nodeIP or localhost

```
# TYPE networkobservability_tcp_connection_remote gauge
networkobservability_tcp_connection_remote{address="0.0.0.0",port="0"} 11
networkobservability_tcp_connection_remote{address="10.224.2.255",port="9090"} 5
networkobservability_tcp_connection_remote{address="10.224.3.2",port="9090"} 2
networkobservability_tcp_connection_remote{address="10.224.3.206",port="8082"} 2
networkobservability_tcp_connection_remote{address="10.224.3.62",port="9090"} 2
networkobservability_tcp_connection_remote{address="104.208.181.172",port="443"} 1
networkobservability_tcp_connection_remote{address="168.63.129.16",port="80"} 1
networkobservability_tcp_connection_remote{address="169.254.169.254",port="80"} 68
networkobservability_tcp_connection_remote{address="20.209.179.65",port="443"} 2
networkobservability_tcp_connection_remote{address="40.64.135.140",port="443"} 0
networkobservability_tcp_connection_remote{address="52.179.73.38",port="443"} 0
networkobservability_tcp_connection_remote{address="52.188.247.151",port="443"} 1
networkobservability_tcp_connection_remote{address="52.253.71.198",port="443"} 5
# HELP networkobservability_tcp_connection_stats TCP connections statistics
```


## Additional Notes

Add any additional notes or context about the pull request here.

---

Please refer to the [CONTRIBUTING.md](../CONTRIBUTING.md) file for more
information on how to contribute to this project.

Author: jimassa

pkg/metrics/metrics.go

@@ -134,13 +134,11 @@ func InitializeMetrics() {
 		exporter.DefaultRegistry,
 		utils.DNSRequestCounterName,
 		dnsRequestCounterDescription,
-		utils.DNSRequestLabels...,
 	)
 	DNSResponseCounter = exporter.CreatePrometheusCounterVecForMetric(
 		exporter.DefaultRegistry,
 		utils.DNSResponseCounterName,
 		dnsResponseCounterDescription,
-		utils.DNSResponseLabels...,
 	)
 
 	// InfiniBand Metrics

pkg/plugin/dns/dns_linux.go

@@ -8,8 +8,6 @@ import (
 	"context"
 	"net"
 	"os"
-	"strconv"
-	"strings"
 
 	v1 "github.com/cilium/cilium/pkg/hubble/api/v1"
 	"github.com/inspektor-gadget/inspektor-gadget/pkg/gadgets/trace/dns/tracer"
@@ -96,18 +94,15 @@ func (d *dns) eventHandler(event *types.Event) {
 	}
 	d.l.Debug("Event received", zap.Any("event", event))
 
-	responses := strings.Join(event.Addresses, ",")
-
-	// Update basic metrics. If the event is a request, the we don't need num_response, response, or return_code
+	// Update basic metrics
 	if event.Qr == types.DNSPktTypeQuery {
 		m = metrics.DNSRequestCounter
-		m.WithLabelValues(event.QType, event.DNSName).Inc()
 	} else if event.Qr == types.DNSPktTypeResponse {
 		m = metrics.DNSResponseCounter
-		m.WithLabelValues(event.Rcode, event.QType, event.DNSName, responses, strconv.Itoa(event.NumAnswers)).Inc()
 	} else {
 		return
 	}
+	m.WithLabelValues().Inc()
 
 	if !d.cfg.EnablePodLevel {
 		return

pkg/plugin/dns/dns_linux_test.go

@@ -141,7 +141,7 @@ func TestRequestEventHandler(t *testing.T) {
 
 	// Basic metrics.
 	mockCV := metrics.NewMockCounterVec(ctrl)
-	mockCV.EXPECT().WithLabelValues(event.QType, event.DNSName).Return(c).Times(1)
+	mockCV.EXPECT().WithLabelValues().Return(c).Times(1)
 	before := value(c)
 	metrics.DNSRequestCounter = mockCV
 
@@ -191,7 +191,7 @@ func TestResponseEventHandler(t *testing.T) {
 	// Basic metrics.
 	c := prometheus.NewCounter(prometheus.CounterOpts{})
 	mockCV := metrics.NewMockCounterVec(ctrl)
-	mockCV.EXPECT().WithLabelValues(event.Rcode, event.QType, event.DNSName, "1.1.1.1,2.2.2.2", "2").Return(c).Times(1)
+	mockCV.EXPECT().WithLabelValues().Return(c).Times(1)
 	before := value(c)
 	metrics.DNSResponseCounter = mockCV
 

pkg/plugin/linuxutil/netstat_stats_linux.go

@@ -22,6 +22,21 @@ const (
 	pathNetSnmp    = "/proc/net/snmp"
 )
 
+var (
+	nodeIP     = os.Getenv("NODE_IP")
+	ignoreList = []string{
+		"0.0.0.0",
+		"127.0.0.",
+	}
+)
+
+func init() {
+	// Add node IP to the ignore list
+	if nodeIP != "" {
+		ignoreList = append(ignoreList, nodeIP)
+	}
+}
+
 type NetstatReader struct {
 	l          *log.ZapLogger
 	connStats  *ConnectionStats
@@ -196,6 +211,9 @@ func (nr *NetstatReader) readSockStats() error {
 				}
 				addr := addrPort.Addr().String()
 				port := strconv.Itoa(int(addrPort.Port()))
+				if !validateRemoteAddr(addr) {
+					continue
+				}
 				// Check if the remote address is in the new sockStats map
 				if _, ok := sockStats.socketByRemoteAddr[remoteAddr]; !ok {
 					nr.l.Debug("Removing remote address from metrics", zap.String("remoteAddr", remoteAddr))
@@ -275,9 +293,11 @@ func validateRemoteAddr(addr string) bool {
 		return false
 	}
 
-	// ignore localhost addresses.
-	if strings.Contains(addr, "127.0.0") {
-		return false
+	// check if the address is in the ignore list
+	for _, addressToIgnore := range ignoreList {
+		if strings.HasPrefix(addr, addressToIgnore) {
+			return false
+		}
 	}
 
 	return true

pkg/plugin/linuxutil/netstat_stats_linux_test.go

@@ -5,6 +5,7 @@ package linuxutil
 import (
 	"fmt"
 	"net"
+	"os"
 	"testing"
 
 	"github.com/cakturk/go-netstat/netstat"
@@ -339,6 +340,9 @@ func TestReadSockStats(t *testing.T) {
 
 // Test IP that belongs to a closed connection being removed from the metrics
 func TestReadSockStatsRemoveClosedConnection(t *testing.T) {
+	// Set os env variable to enable debug logs
+	os.Setenv("NODE_IP", "10.224.0.6")
+
 	_, err := log.SetupZapLogger(log.GetDefaultLogOpts())
 	require.NoError(t, err)
 	opts := &NetstatOpts{
@@ -363,7 +367,7 @@ func TestReadSockStatsRemoveClosedConnection(t *testing.T) {
 	// Initial value
 	nr.opts.PrevTCPSockStats = &SocketStats{
 		socketByRemoteAddr: map[string]int{
-			"127.0.0.1:80": 1,
+			"192.168.1.100:80": 1,
 		},
 	}
 
@@ -372,7 +376,11 @@ func TestReadSockStatsRemoveClosedConnection(t *testing.T) {
 	ns.EXPECT().UDPSocks(gomock.Any()).Return([]netstat.SockTabEntry{}, nil).Times(1)
 
 	// We are expecting the gauge to be called once for this value as it is removed
-	MockGaugeVec.EXPECT().WithLabelValues("127.0.0.1", "80").Return(testmetric).Times(1)
+	MockGaugeVec.EXPECT().WithLabelValues("192.168.1.100", "80").Return(testmetric).Times(1)
+
+	// We are not expecting the gauge to be called for localhost or node IP
+	MockGaugeVec.EXPECT().WithLabelValues("127.0.0.1", "80").Return(testmetric).Times(0)
+	MockGaugeVec.EXPECT().WithLabelValues("10.224.0.6", "80").Return(testmetric).Times(0)
 
 	err = nr.readSockStats()
 	require.NoError(t, err)

test/e2e/framework/prometheus/prometheus.go

@@ -83,6 +83,12 @@ func verifyValidMetricPresent(metricName string, data map[string]*promclient.Met
 				for _, label := range metric.GetLabel() {
 					metricLabels[label.GetName()] = label.GetValue()
 				}
+
+				// if valid metric is empty, then we just need to make sure the metric and value is present
+				if len(validMetric) == 0 && len(metricLabels) > 0 {
+					return nil
+				}
+
 				if reflect.DeepEqual(metricLabels, validMetric) {
 					return nil
 				}

test/e2e/scenarios/dns/validate-basic-dns-metric.go

@@ -24,10 +24,7 @@ type validateBasicDNSRequestMetrics struct {
 func (v *validateBasicDNSRequestMetrics) Run() error {
 	metricsEndpoint := fmt.Sprintf("http://localhost:%d/metrics", common.RetinaPort)
 
-	validBasicDNSRequestMetricLabels := map[string]string{
-		"query":      v.Query,
-		"query_type": v.QueryType,
-	}
+	validBasicDNSRequestMetricLabels := map[string]string{}
 
 	err := prom.CheckMetric(metricsEndpoint, dnsBasicRequestCountMetricName, validBasicDNSRequestMetricLabels)
 	if err != nil {
@@ -61,13 +58,7 @@ func (v *validateBasicDNSResponseMetrics) Run() error {
 		v.Response = ""
 	}
 
-	validBasicDNSResponseMetricLabels := map[string]string{
-		"num_response": v.NumResponse,
-		"query":        v.Query,
-		"query_type":   v.QueryType,
-		"return_code":  v.ReturnCode,
-		"response":     v.Response,
-	}
+	validBasicDNSResponseMetricLabels := map[string]string{}
 
 	err := prom.CheckMetric(metricsEndpoint, dnsBasicResponseCountMetricName, validBasicDNSResponseMetricLabels)
 	if err != nil {

test/e2e/scenarios/tcp/validate-tcp-connection-remote.go

@@ -21,9 +21,7 @@ type ValidateRetinaTCPConnectionRemoteMetric struct {
 func (v *ValidateRetinaTCPConnectionRemoteMetric) Run() error {
 	promAddress := fmt.Sprintf("http://localhost:%s/metrics", v.PortForwardedRetinaPort)
 
-	validMetrics := []map[string]string{
-		{address: "0.0.0.0", port: "0"},
-	}
+	validMetrics := []map[string]string{}
 
 	for _, metric := range validMetrics {
 		err := prom.CheckMetric(promAddress, tcpConnectionRemoteMetricName, metric)
@@ -32,7 +30,7 @@ func (v *ValidateRetinaTCPConnectionRemoteMetric) Run() error {
 		}
 	}
 
-	log.Printf("found metrics matching %+v\n", validMetrics)
+	log.Printf("found metrics matching %+v\n", tcpConnectionRemoteMetricName)
 	return nil
 }