Set DSTACK_OTEL_METRICS_EXPORTERS=otlp by default

Author: r4victor

mkdocs/docs/guides/server-deployment.md

@@ -444,11 +444,14 @@ $ DSTACK_OTEL_TRACES_ENABLED=1 \
     - `DSTACK_OTEL_TRACES_BACKGROUND_SAMPLE_RATE` – The head sampling rate for background task traces.
 
 ??? info "Metrics exporters"
-    By default, if the [Prometheus `/metrics` endpoint](../concepts/metrics.md) is enabled via
-    `DSTACK_ENABLE_PROMETHEUS_METRICS`, OpenTelemetry metrics are exposed there alongside the built-in
-    `dstack` metrics; otherwise they are pushed via OTLP. Set `DSTACK_OTEL_METRICS_EXPORTERS`
-    to a comma-separated list of `prometheus` and/or `otlp` to override, e.g. force OTLP push
-    even when `/metrics` is enabled.
+    By default, metrics are pushed via OTLP like traces and logs. Set
+    `DSTACK_OTEL_METRICS_EXPORTERS=prometheus` to instead expose them on the
+    [Prometheus `/metrics` endpoint](../concepts/metrics.md) alongside the built-in `dstack`
+    metrics (a comma-separated list enables both exporters at once).
+
+    If you run multiple server replicas, only use the `prometheus` exporter if Prometheus
+    scrapes each replica directly. Scraping through a load balancer interleaves the
+    replicas' counters into the same series and silently corrupts all rates.
 
 ## Encryption
 

mkdocs/docs/reference/env.md

@@ -130,7 +130,7 @@ For more details on the options below, refer to the [server deployment](../guide
 - `DSTACK_OTEL_TRACES_BACKGROUND_SAMPLE_RATE`{ #DSTACK_OTEL_TRACES_BACKGROUND_SAMPLE_RATE } – The head sampling rate for background task traces. Defaults to `1.0`.
 - `DSTACK_OTEL_LOGS_ENABLED`{ #DSTACK_OTEL_LOGS_ENABLED } – Enables server log export via OTLP if set to any value. Requires the `otel` extra.
 - `DSTACK_OTEL_METRICS_ENABLED`{ #DSTACK_OTEL_METRICS_ENABLED } – Enables OpenTelemetry metrics if set to any value. Requires the `otel` extra.
-- `DSTACK_OTEL_METRICS_EXPORTERS`{ #DSTACK_OTEL_METRICS_EXPORTERS } – A comma-separated list of OpenTelemetry metrics exporters: `prometheus` (expose on the `/metrics` endpoint) and/or `otlp` (push via OTLP). Defaults to `prometheus` if `DSTACK_ENABLE_PROMETHEUS_METRICS` is set, otherwise `otlp`.
+- `DSTACK_OTEL_METRICS_EXPORTERS`{ #DSTACK_OTEL_METRICS_EXPORTERS } – A comma-separated list of OpenTelemetry metrics exporters: `otlp` (push via OTLP) and/or `prometheus` (expose on the `/metrics` endpoint). Defaults to `otlp`.
 - `DSTACK_DEFAULT_SERVICE_CLIENT_MAX_BODY_SIZE`{ #DSTACK_DEFAULT_SERVICE_CLIENT_MAX_BODY_SIZE } – Request body size limit for services running with a gateway, in bytes. Defaults to 64 MiB.
 - `DSTACK_SERVICE_CLIENT_TIMEOUT`{ #DSTACK_SERVICE_CLIENT_TIMEOUT } – Timeout in seconds for HTTP requests sent from the in-server proxy and gateways to service replicas. Defaults to 60.
 - `DSTACK_FORBID_SERVICES_WITHOUT_GATEWAY`{ #DSTACK_FORBID_SERVICES_WITHOUT_GATEWAY } – Forbids registering new services without a gateway if set to any value.

src/dstack/_internal/server/identity.py

@@ -0,0 +1,7 @@
+import uuid
+
+SERVER_REPLICA_ID = str(uuid.uuid4())
+"""
+Unique ID of this server process, regenerated on every start.
+Distinguishes replicas in multi-replica deployments, e.g. in exported telemetry.
+"""

src/dstack/_internal/server/settings.py

@@ -136,9 +136,9 @@
 OTEL_METRICS_ENABLED = os.getenv("DSTACK_OTEL_METRICS_ENABLED") is not None
 """Enables OpenTelemetry metrics. Requires the `otel` extra to be installed."""
 OTEL_METRICS_EXPORTERS = os.getenv("DSTACK_OTEL_METRICS_EXPORTERS")
-"""A comma-separated list of metrics exporters: `prometheus` (expose via the /metrics endpoint)
-and/or `otlp` (push via OTLP, configured by standard `OTEL_*` env vars).
-Defaults to `prometheus` if the /metrics endpoint is enabled, otherwise `otlp`.
+"""A comma-separated list of metrics exporters: `otlp` (push via OTLP, configured by
+standard `OTEL_*` env vars) and/or `prometheus` (expose via the /metrics endpoint).
+Defaults to `otlp`.
 """
 
 DEFAULT_CREDS_DISABLED = os.getenv("DSTACK_DEFAULT_CREDS_DISABLED") is not None

src/dstack/_internal/server/utils/otel/utils.py

@@ -37,6 +37,7 @@
 
 from dstack._internal import settings as core_settings
 from dstack._internal.server import settings
+from dstack._internal.server.identity import SERVER_REPLICA_ID
 from dstack._internal.server.utils.common import is_background_task_name
 from dstack._internal.server.utils.logging import AsyncioCancelledErrorFilter
 from dstack._internal.utils.logging import get_logger
@@ -110,10 +111,12 @@ def _instrument(app: FastAPI, engine: AsyncEngine) -> None:
 
 
 def _get_metrics_exporters() -> List[str]:
+    # OTLP by default, like traces and logs. The prometheus exporter (metrics on
+    # the /metrics endpoint) is opt-in: scraping /metrics through a load
+    # balancer with multiple server replicas interleaves the replicas' counters
+    # into the same series, silently corrupting rates.
     if settings.OTEL_METRICS_EXPORTERS is not None:
         return [e.strip() for e in settings.OTEL_METRICS_EXPORTERS.split(",") if e.strip()]
-    if settings.ENABLE_PROMETHEUS_METRICS:
-        return ["prometheus"]
     return ["otlp"]
 
 
@@ -137,6 +140,9 @@ def _get_resource() -> Resource:
     return Resource.create(
         {
             "service.name": "dstack-server",
+            # Distinguishes replicas: without it, metrics pushed by multiple
+            # server replicas collapse into the same series, corrupting counters
+            "service.instance.id": SERVER_REPLICA_ID,
             "service.version": core_settings.DSTACK_VERSION or "dev",
             "deployment.environment.name": settings.SERVER_ENVIRONMENT,
         }

src/tests/_internal/server/utils/test_tracing.py

@@ -148,19 +148,17 @@ def test_does_not_export_otel_sdk_records(self):
 
 class TestGetMetricsExporters:
     @pytest.mark.parametrize(
-        ("exporters", "prometheus_enabled", "expected"),
+        ("exporters", "expected"),
         [
-            (None, True, ["prometheus"]),
-            (None, False, ["otlp"]),
-            ("otlp", True, ["otlp"]),
-            ("prometheus, otlp", False, ["prometheus", "otlp"]),
+            (None, ["otlp"]),
+            ("prometheus", ["prometheus"]),
+            ("prometheus, otlp", ["prometheus", "otlp"]),
         ],
     )
-    def test_returns_expected(self, monkeypatch, exporters, prometheus_enabled, expected):
+    def test_returns_expected(self, monkeypatch, exporters, expected):
         from dstack._internal.server import settings
 
         monkeypatch.setattr(settings, "OTEL_METRICS_EXPORTERS", exporters)
-        monkeypatch.setattr(settings, "ENABLE_PROMETHEUS_METRICS", prometheus_enabled)
         assert _get_metrics_exporters() == expected