Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion openapi/ga/individual/platform.openapi.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

11 changes: 10 additions & 1 deletion openapi/ga/openapi.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

11 changes: 10 additions & 1 deletion openapi/openapi.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions packages/nemo_platform/pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,7 @@ jobs-service = [
"pyyaml>=6.0.2",
"hvac>=2.3.0",
"nmp-common",
"nemo-platform-plugin",
"aiosqlite>=0.20.0",
"duckdb<2.0.0,>=1.1.3",
"pandas>=1.5.3",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import httpx
import typer
from nemo_platform_ext.cli.core.help_formatter import create_typer_app
from nemo_platform_ext.cli.docker_preflight import require_docker_for_default_local
from nemo_platform_ext.local.process import (
ForegroundInstanceError,
InstanceAlreadyRunningError,
Expand Down Expand Up @@ -236,16 +237,6 @@ def run_services(

_ensure_port_available(host, port, scope, base_dir=base_dir)

try:
lock_fd = acquire_lock(scope, base_dir=base_dir)
except InstanceAlreadyRunningError:
_fail_already_running(scope, base_dir)

# _NMP_LAUNCH_MODE is set by start_background() when this process was
# spawned via ``nemo services start``. Without it we default to
# "foreground", which protects interactive ``run`` sessions from being
# killed by ``stop``.
mode = "background" if os.environ.get("_NMP_LAUNCH_MODE") == "background" else "foreground"
platform_config = PlatformAppConfig(
services=_parse_csv_option(services),
service_group=service_group,
Expand All @@ -259,6 +250,18 @@ def run_services(
keep_alive_timeout_seconds=keep_alive_timeout_seconds,
state_root=base_dir,
)
require_docker_for_default_local(platform_config)

try:
lock_fd = acquire_lock(scope, base_dir=base_dir)
except InstanceAlreadyRunningError:
_fail_already_running(scope, base_dir)

# _NMP_LAUNCH_MODE is set by start_background() when this process was
# spawned via ``nemo services start``. Without it we default to
# "foreground", which protects interactive ``run`` sessions from being
# killed by ``stop``.
mode = "background" if os.environ.get("_NMP_LAUNCH_MODE") == "background" else "foreground"

desc = InstanceDescriptor.from_config(
platform_config,
Expand Down Expand Up @@ -382,6 +385,7 @@ def start_services(
keep_alive_timeout_seconds=keep_alive_timeout_seconds,
state_root=base_dir,
)
require_docker_for_default_local(platform_config)

typer.echo("Starting platform services...")
proc = start_background(platform_config)
Expand Down Expand Up @@ -565,11 +569,6 @@ def restart_services(
)
raise typer.Exit(1)

typer.echo("Stopping platform services...")
# restart always produces a background instance, so force=True is
# appropriate even for foreground targets.
stop_instance(scope, base_dir=base_dir, force=True)

previous_config = prev.config if prev else None
effective_services = _parse_csv_option(services) if services is not None else None
if services is None and previous_config is not None:
Expand Down Expand Up @@ -601,9 +600,6 @@ def restart_services(
)
)

_warn_bind_all(effective_host)

_ensure_port_available(effective_host, effective_port, scope, base_dir=base_dir)
platform_config = PlatformAppConfig(
services=effective_services,
service_group=effective_service_group,
Expand All @@ -617,6 +613,17 @@ def restart_services(
keep_alive_timeout_seconds=effective_keep_alive_timeout_seconds,
state_root=base_dir,
)
# Preflight before stop so a missing Docker daemon does not tear down a healthy instance.
require_docker_for_default_local(platform_config)

typer.echo("Stopping platform services...")
# restart always produces a background instance, so force=True is
# appropriate even for foreground targets.
stop_instance(scope, base_dir=base_dir, force=True)

_warn_bind_all(effective_host)

_ensure_port_available(effective_host, effective_port, scope, base_dir=base_dir)

typer.echo("Starting platform services...")
proc = start_background(platform_config)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@
import typer
import yaml as _yaml
from nemo_platform import NeMoPlatform
from nemo_platform_plugin.capabilities import probe_docker
from nemo_platform_plugin.client.adapter import client_from_platform
from nemo_platform_plugin.config import validate_docker_available
from nemo_platform_plugin.secrets.client import SecretsClient
from nemo_platform_plugin.secrets.types import PlatformSecretCreateRequest, PlatformSecretUpdateRequest
from nmp.common.config import nmp_user_data_dir
Expand All @@ -43,6 +43,7 @@
from nemo_platform_ext.cli.commands.skills.registry import get_installer, load_skills
from nemo_platform_ext.cli.core.context import CLIContext
from nemo_platform_ext.cli.core.errors import handle_errors
from nemo_platform_ext.cli.docker_preflight import DOCKER_PREFLIGHT_MESSAGE, require_docker_for_default_local
from nemo_platform_ext.cli.telemetry import emit
from nemo_platform_ext.cli.telemetry.events import OnboardingStepEvent, TaskStatusEnum
from nemo_platform_ext.client.tls import client_verify_from_env
Expand Down Expand Up @@ -1057,7 +1058,7 @@ def _should_hint_docker_unavailable(*, exit_code: int | None, log_path: Path | N
"""
if _services_log_suggests_docker_failure(log_path):
return True
if exit_code is not None and not validate_docker_available():
if exit_code is not None and not probe_docker(use_cache=False).available:
return True
return False

Expand Down Expand Up @@ -1158,6 +1159,9 @@ def _maybe_start_services(
console.print(" [cyan]PYO3_USE_ABI3_FORWARD_COMPATIBILITY=1 pip install 'nemo-platform\\[all]'[/cyan]")
raise typer.Exit(1)

# Fail before stop/spawn when default local needs Docker (NVBug 6537617).
require_docker_for_default_local(console=console)

if already_running:
console.print(" Restarting platform services...")
_kill_existing_services(base_url)
Expand Down Expand Up @@ -1186,10 +1190,7 @@ def _maybe_start_services(
console.print(f"{CROSS} Platform did not become ready within {timeout}s")
console.print(f" Check {log} for details.")
if _should_hint_docker_unavailable(exit_code=exit_code, log_path=log):
console.print(
" Docker does not appear to be available. "
"Install and start Docker, or configure non-Docker executors, then retry."
)
console.print(f" {DOCKER_PREFLIGHT_MESSAGE}")
raise typer.Exit(1)

console.print(f"{CHECK} Platform running at {base_url} (pid {proc.pid})\n")
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,147 @@
# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

"""Config-aware Docker preflight for local platform startup (NVBug 6537617).

Fails fast before spawn/wait when the resolved run would start the deployments
service/controller with a docker-backed ``default_executor`` while the daemon is
unreachable. Does not treat Docker as a global platform dependency — kubernetes and
reduced selections that never hit that fail-close path are left alone.
"""

from __future__ import annotations

from dataclasses import dataclass, replace
from pathlib import Path
from typing import Any

import typer
import yaml
from nemo_platform_plugin.capabilities import probe_docker
from nmp.platform_runner.config import PlatformAppConfig, default_config_path, resolve_run_configuration
from rich.console import Console

DOCKER_PREFLIGHT_MESSAGE = (
"Docker is required for this default local setup (deployments default_executor "
"uses the docker backend) but the Docker daemon is not available. "
"Install and start Docker, or use a kubernetes / non-docker config "
"(and omit the deployments service if you do not need it), then retry."
)


@dataclass(frozen=True)
class _DefaultDockerExecutorProbe:
"""Whether the default deployments executor is docker, and its optional host override."""

is_docker: bool
docker_host: str | None = None


def _load_platform_yaml(config_path: str) -> dict[str, Any]:
"""Load platform YAML without running Pydantic validators (no soft-downgrade)."""
path = Path(config_path)
if not path.is_file():
return {}
with path.open(encoding="utf-8") as fh:
data = yaml.safe_load(fh) or {}
return data if isinstance(data, dict) else {}


def _intended_runtime_is_kubernetes(raw: dict[str, Any]) -> bool:
platform = raw.get("platform")
if not isinstance(platform, dict):
return False
runtime = platform.get("runtime")
return isinstance(runtime, str) and runtime.strip().lower() == "kubernetes"


def _default_docker_executor_probe(raw: dict[str, Any]) -> _DefaultDockerExecutorProbe:
"""Resolve deployments.default_executor → docker backend + optional config.docker_host."""
deployments = raw.get("deployments")
if not isinstance(deployments, dict):
return _DefaultDockerExecutorProbe(is_docker=False)
default_name = deployments.get("default_executor")
if not isinstance(default_name, str) or not default_name:
return _DefaultDockerExecutorProbe(is_docker=False)
executors = deployments.get("executors")
if not isinstance(executors, list):
return _DefaultDockerExecutorProbe(is_docker=False)
for spec in executors:
if not isinstance(spec, dict):
continue
if spec.get("name") != default_name:
continue
backend = spec.get("backend")
if not (isinstance(backend, str) and backend.strip().lower() == "docker"):
return _DefaultDockerExecutorProbe(is_docker=False)
docker_host: str | None = None
config = spec.get("config")
if isinstance(config, dict):
host = config.get("docker_host")
if isinstance(host, str) and host.strip():
docker_host = host.strip()
return _DefaultDockerExecutorProbe(is_docker=True, docker_host=docker_host)
return _DefaultDockerExecutorProbe(is_docker=False)


def _resolve_default_local_docker_probe(
platform_config: PlatformAppConfig | None = None,
*,
config_path: str | None = None,
) -> _DefaultDockerExecutorProbe | None:
"""Return the docker probe target when this run needs the default-local gate; else None."""
app_config = platform_config or PlatformAppConfig()
if config_path is not None:
app_config = replace(app_config, config_path=config_path)

try:
resolved = resolve_run_configuration(app_config)
except ValueError:
# Invalid selections fail elsewhere; do not block on Docker here.
return None

starts_deployments = "deployments" in resolved.services or "deployments" in resolved.controllers
if not starts_deployments:
return None

raw = _load_platform_yaml(resolved.config_path or default_config_path())
if _intended_runtime_is_kubernetes(raw):
return None

probe_target = _default_docker_executor_probe(raw)
if not probe_target.is_docker:
return None
return probe_target


def default_local_needs_docker(
platform_config: PlatformAppConfig | None = None,
*,
config_path: str | None = None,
) -> bool:
"""Return whether this run would fail closed on a missing Docker daemon."""
return _resolve_default_local_docker_probe(platform_config, config_path=config_path) is not None


def require_docker_for_default_local(
platform_config: PlatformAppConfig | None = None,
*,
config_path: str | None = None,
console: Console | None = None,
) -> None:
"""Exit with a clear message when default-local Docker is required but missing."""
probe_target = _resolve_default_local_docker_probe(platform_config, config_path=config_path)
if probe_target is None:
return
if probe_docker(docker_host=probe_target.docker_host, use_cache=False).available:
return
out = console or Console(stderr=True)
out.print(f"[red]✗[/red] {DOCKER_PREFLIGHT_MESSAGE}")
raise typer.Exit(1)


__all__ = [
"DOCKER_PREFLIGHT_MESSAGE",
"default_local_needs_docker",
"require_docker_for_default_local",
]
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@
from dataclasses import dataclass
from enum import Enum

from nemo_platform_plugin.capabilities import probe_docker

from .config import QuickstartConfig


Expand Down Expand Up @@ -96,25 +98,23 @@ def run_all(self) -> list[PreflightResult]:

def _check_docker_available(self) -> None:
"""Verify Docker daemon is running and accessible."""
try:
import docker

client = docker.from_env()
client.ping()
# Preflight can be re-run after the user starts Docker.
result = probe_docker(use_cache=False)
if result.available:
self.results.append(
PreflightResult(
name="Docker Available",
status=CheckStatus.PASS,
message="Docker daemon is running",
)
)
except Exception as e:
else:
self.results.append(
PreflightResult(
name="Docker Available",
status=CheckStatus.FAIL,
message="Docker daemon is not accessible",
details=str(e),
details=result.detail or "Docker daemon unreachable",
)
)

Expand Down
Loading
Loading