Show host GPU driver on fleet instances

Currently, the only way to know which GPU driver a fleet instance runs is to
provision it and run a task there.

Now shim detects the host GPU driver version on start (`nvidia-smi` for NVIDIA,
`amd-smi` for AMD, `/sys/module/tenstorrent/version` for Tenstorrent) and reports
it via the new `GET /api/instance/info` endpoint. The server stores it in
`JobProvisioningData`, so that no migration is needed, and exposes it as
`Instance.gpu_driver` and as a DRIVER column in `dstack fleet -v`.

Notes:

- Instance info reports facts observed by shim, as opposed to `/api/components`,
  which reports software managed by shim.
- GPU hosts are asked on every instance check, as the facts change when shim
  restarts, e.g., after a driver upgrade, which the server does not necessarily
  observe. The provisioning data is only updated if the driver changed.
- SSH fleets are also asked at deploy time, so that the driver is known as soon
  as the instance is ready.
- The driver stays unknown on hosts without GPUs, if detection fails, and on
  backends where shim does not run (runpod, vastai, kubernetes).

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: Andrey Cheptsov

runner/docs/shim.openapi.yaml

@@ -2,7 +2,7 @@ openapi: 3.1.2
 
 info:
   title: dstack-shim API
-  version: v2/0.20.1
+  version: v2/0.20.30
   x-logo:
     url: https://avatars.githubusercontent.com/u/54146142?s=260
   description: >
@@ -86,6 +86,22 @@ paths:
               schema:
                 $ref: "#/components/schemas/InstanceHealthResponse"
 
+  /instance/info:
+    get:
+      summary: Get instance info
+      description: >
+        (since [0.20.30](https://github.com/dstackai/dstack/releases/tag/0.20.30))
+        Returns facts about the host observed by shim, e.g., the GPU driver version.
+        Unlike `/components`, the reported entities are not managed by shim.
+      tags: [Instance]
+      responses:
+        "200":
+          description: ""
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/InstanceInfoResponse"
+
   /components:
     get:
       summary: Get components
@@ -504,6 +520,24 @@ components:
           $ref: "#/components/schemas/DCGMHealth"
       additionalProperties: false
 
+    InstanceInfoResponse:
+      title: shim.api.InstanceInfoResponse
+      type: object
+      properties:
+        gpu_vendor:
+          description: Host GPU vendor. Omitted on hosts without GPUs.
+          type: string
+          examples:
+            - nvidia
+        gpu_driver_version:
+          description: >
+            Host GPU driver version. Omitted on hosts without GPUs
+            or if detection failed.
+          type: string
+          examples:
+            - 570.86.15
+      additionalProperties: false
+
     ComponentListResponse:
       title: shim.api.ComponentListResponse
       type: object

runner/internal/shim/api/api_test.go

@@ -5,10 +5,12 @@ import (
 	"sync"
 
 	"github.com/dstackai/dstack/runner/internal/shim"
+	"github.com/dstackai/dstack/runner/internal/shim/host"
 )
 
 type DummyRunner struct {
 	tasks map[string]bool
+	gpus  []host.GpuInfo
 	mu    sync.Mutex
 }
 
@@ -46,6 +48,10 @@ func (ds *DummyRunner) Resources(context.Context) shim.Resources {
 	return shim.Resources{}
 }
 
+func (ds *DummyRunner) Gpus(context.Context) []host.GpuInfo {
+	return ds.gpus
+}
+
 func NewDummyRunner() *DummyRunner {
 	return &DummyRunner{
 		tasks: map[string]bool{},

runner/internal/shim/api/handlers.go

@@ -51,6 +51,18 @@ func (s *ShimServer) InstanceHealthHandler(w http.ResponseWriter, r *http.Reques
 	return &response, nil
 }
 
+func (s *ShimServer) InstanceInfoHandler(w http.ResponseWriter, r *http.Request) (interface{}, error) {
+	response := InstanceInfoResponse{}
+	// GPUs are detected once on startup, so this is not an expensive call.
+	// The driver is a host-wide property, hence any GPU can be used as the source.
+	if gpus := s.runner.Gpus(r.Context()); len(gpus) > 0 {
+		response.GpuVendor = string(gpus[0].Vendor)
+		response.GpuDriverVersion = gpus[0].DriverVersion
+	}
+
+	return &response, nil
+}
+
 func (s *ShimServer) TaskListHandler(w http.ResponseWriter, r *http.Request) (interface{}, error) {
 	tasks := s.runner.TaskList()
 	return &TaskListResponse{tasks}, nil

runner/internal/shim/api/handlers_test.go

@@ -7,6 +7,8 @@ import (
 	"testing"
 
 	commonapi "github.com/dstackai/dstack/runner/internal/common/api"
+	"github.com/dstackai/dstack/runner/internal/common/gpu"
+	"github.com/dstackai/dstack/runner/internal/shim/host"
 )
 
 func TestHealthcheck(t *testing.T) {
@@ -29,6 +31,50 @@ func TestHealthcheck(t *testing.T) {
 	}
 }
 
+// TestInstanceInfo goes through the router to also cover the endpoint registration
+func TestInstanceInfo(t *testing.T) {
+	request := httptest.NewRequest("GET", "/api/instance/info", nil)
+	responseRecorder := httptest.NewRecorder()
+
+	runner := NewDummyRunner()
+	runner.gpus = []host.GpuInfo{
+		{Vendor: gpu.GpuVendorNvidia, Name: "T4", Vram: 16384, DriverVersion: "570.86.15"},
+	}
+	server := NewShimServer(context.Background(), ":12346", "0.0.1.dev2", runner, nil, nil, nil, nil)
+
+	server.httpServer.Handler.ServeHTTP(responseRecorder, request)
+
+	if responseRecorder.Code != 200 {
+		t.Errorf("Want status '%d', got '%d'", 200, responseRecorder.Code)
+	}
+
+	expected := `{"gpu_vendor":"nvidia","gpu_driver_version":"570.86.15"}`
+
+	if strings.TrimSpace(responseRecorder.Body.String()) != expected {
+		t.Errorf("Want '%s', got '%s'", expected, responseRecorder.Body.String())
+	}
+}
+
+func TestInstanceInfoWithoutGpus(t *testing.T) {
+	request := httptest.NewRequest("GET", "/api/instance/info", nil)
+	responseRecorder := httptest.NewRecorder()
+
+	server := NewShimServer(context.Background(), ":12347", "0.0.1.dev2", NewDummyRunner(), nil, nil, nil, nil)
+
+	f := commonapi.JSONResponseHandler(server.InstanceInfoHandler)
+	f(responseRecorder, request)
+
+	if responseRecorder.Code != 200 {
+		t.Errorf("Want status '%d', got '%d'", 200, responseRecorder.Code)
+	}
+
+	expected := "{}"
+
+	if strings.TrimSpace(responseRecorder.Body.String()) != expected {
+		t.Errorf("Want '%s', got '%s'", expected, responseRecorder.Body.String())
+	}
+}
+
 func TestTaskSubmit(t *testing.T) {
 	server := NewShimServer(context.Background(), ":12340", "0.0.1.dev2", NewDummyRunner(), nil, nil, nil, nil)
 	requestBody := `{

runner/internal/shim/api/schemas.go

@@ -19,6 +19,13 @@ type InstanceHealthResponse struct {
 	DCGM *dcgm.Health `json:"dcgm"`
 }
 
+// InstanceInfoResponse reports facts about the host observed by shim. Fields are
+// omitted if the corresponding fact is not applicable or could not be detected.
+type InstanceInfoResponse struct {
+	GpuVendor        string `json:"gpu_vendor,omitempty"`
+	GpuDriverVersion string `json:"gpu_driver_version,omitempty"`
+}
+
 type TaskListResponse struct {
 	Tasks []*shim.TaskListItem `json:"tasks"`
 }

runner/internal/shim/api/server.go

@@ -13,6 +13,7 @@ import (
 	"github.com/dstackai/dstack/runner/internal/shim"
 	"github.com/dstackai/dstack/runner/internal/shim/components"
 	"github.com/dstackai/dstack/runner/internal/shim/dcgm"
+	"github.com/dstackai/dstack/runner/internal/shim/host"
 )
 
 type TaskRunner interface {
@@ -22,6 +23,7 @@ type TaskRunner interface {
 	Remove(ctx context.Context, taskID string) error
 
 	Resources(context.Context) shim.Resources
+	Gpus(context.Context) []host.GpuInfo
 	TaskList() []*shim.TaskListItem
 	TaskInfo(taskID string) shim.TaskInfo
 }
@@ -85,6 +87,7 @@ func NewShimServer(
 	r.AddHandler("GET", "/api/healthcheck", s.HealthcheckHandler)
 	r.AddHandler("POST", "/api/shutdown", s.ShutdownHandler)
 	r.AddHandler("GET", "/api/instance/health", s.InstanceHealthHandler)
+	r.AddHandler("GET", "/api/instance/info", s.InstanceInfoHandler)
 	r.AddHandler("GET", "/api/components", s.ComponentListHandler)
 	r.AddHandler("POST", "/api/components/install", s.ComponentInstallHandler)
 	r.AddHandler("GET", "/api/tasks", s.TaskListHandler)

runner/internal/shim/docker.go

@@ -324,6 +324,12 @@ func (d *DockerRunner) Resources(ctx context.Context) Resources {
 	}
 }
 
+// Gpus returns the GPUs detected at startup without collecting other host
+// resources, making it suitable for frequently called paths.
+func (d *DockerRunner) Gpus(ctx context.Context) []host.GpuInfo {
+	return d.gpus
+}
+
 func (d *DockerRunner) TaskList() []*TaskListItem {
 	tasks := d.tasks.List()
 	result := make([]*TaskListItem, 0, len(tasks))