diff --git a/extensions/fine_python_ast/pyproject.toml b/extensions/fine_python_ast/pyproject.toml index ead7e05..31ff2e7 100644 --- a/extensions/fine_python_ast/pyproject.toml +++ b/extensions/fine_python_ast/pyproject.toml @@ -6,3 +6,15 @@ authors = [{ name = "Vladyslav Hnatiuk", email = "aders1234@gmail.com" }] readme = "README.md" requires-python = ">=3.11, < 3.14" dependencies = ["finecode_extension_api==0.3.*"] + +[dependency-groups] +dev_workspace = ["finecode==0.3.*", "finecode_dev_common_preset==0.2.*"] + +[tool.finecode] +presets = [{ source = "finecode_dev_common_preset" }] + +[tool.finecode.env.dev_workspace.dependencies] +finecode_dev_common_preset = { path = "../../finecode_dev_common_preset", editable = true } +finecode = { path = "../../", editable = true } +finecode_extension_runner = { path = "../../finecode_extension_runner", editable = true } +finecode_extension_api = { path = "../../finecode_extension_api", editable = true } diff --git a/extensions/fine_python_black/pyproject.toml b/extensions/fine_python_black/pyproject.toml index 779196f..c0f808b 100644 --- a/extensions/fine_python_black/pyproject.toml +++ b/extensions/fine_python_black/pyproject.toml @@ -6,3 +6,15 @@ authors = [{ name = "Vladyslav Hnatiuk", email = "aders1234@gmail.com" }] readme = "README.md" requires-python = ">=3.11, < 3.14" dependencies = ["finecode_extension_api==0.3.*", "black (>=25.1.0,<26.0.0)"] + +[dependency-groups] +dev_workspace = ["finecode==0.3.*", "finecode_dev_common_preset==0.2.*"] + +[tool.finecode] +presets = [{ source = "finecode_dev_common_preset" }] + +[tool.finecode.env.dev_workspace.dependencies] +finecode_dev_common_preset = { path = "../../finecode_dev_common_preset", editable = true } +finecode = { path = "../../", editable = true } +finecode_extension_runner = { path = "../../finecode_extension_runner", editable = true } +finecode_extension_api = { path = "../../finecode_extension_api", editable = true } diff --git a/extensions/fine_python_import_linter/pyproject.toml b/extensions/fine_python_import_linter/pyproject.toml index 16558e3..bca107a 100644 --- a/extensions/fine_python_import_linter/pyproject.toml +++ b/extensions/fine_python_import_linter/pyproject.toml @@ -5,12 +5,21 @@ description = "" authors = [{ name = "Vladyslav Hnatiuk", email = "aders1234@gmail.com" }] readme = "README.md" requires-python = ">= 3.11, < 3.14" -dependencies = [ - "finecode_extension_api @ git+https://github.com/finecode-dev/finecode.git#subdirectory=finecode_extension_api", - "import-linter (>=2.1,<3.0)", -] +dependencies = ["finecode_extension_api == 0.3.*", "import-linter (>=2.1,<3.0)"] [build-system] requires = ["poetry-core>=2.0.0,<3.0.0"] build-backend = "poetry.core.masonry.api" + +[dependency-groups] +dev_workspace = ["finecode==0.3.*", "finecode_dev_common_preset==0.2.*"] + +[tool.finecode] +presets = [{ source = "finecode_dev_common_preset" }] + +[tool.finecode.env.dev_workspace.dependencies] +finecode_dev_common_preset = { path = "../../finecode_dev_common_preset", editable = true } +finecode = { path = "../../", editable = true } +finecode_extension_runner = { path = "../../finecode_extension_runner", editable = true } +finecode_extension_api = { path = "../../finecode_extension_api", editable = true } diff --git a/extensions/fine_python_isort/pyproject.toml b/extensions/fine_python_isort/pyproject.toml index d0c8722..6fe8bd9 100644 --- a/extensions/fine_python_isort/pyproject.toml +++ b/extensions/fine_python_isort/pyproject.toml @@ -6,3 +6,15 @@ authors = [{ name = "Vladyslav Hnatiuk", email = "aders1234@gmail.com" }] readme = "README.md" requires-python = ">= 3.11, < 3.14" dependencies = ["finecode_extension_api==0.3.*", "isort (>=5.13, <6)"] + +[dependency-groups] +dev_workspace = ["finecode==0.3.*", "finecode_dev_common_preset==0.2.*"] + +[tool.finecode] +presets = [{ source = "finecode_dev_common_preset" }] + +[tool.finecode.env.dev_workspace.dependencies] +finecode_dev_common_preset = { path = "../../finecode_dev_common_preset", editable = true } +finecode = { path = "../../", editable = true } +finecode_extension_runner = { path = "../../finecode_extension_runner", editable = true } +finecode_extension_api = { path = "../../finecode_extension_api", editable = true } diff --git a/extensions/fine_python_module_exports/pyproject.toml b/extensions/fine_python_module_exports/pyproject.toml index 85b4348..f41f6f9 100644 --- a/extensions/fine_python_module_exports/pyproject.toml +++ b/extensions/fine_python_module_exports/pyproject.toml @@ -6,3 +6,15 @@ authors = [{ name = "Vladyslav Hnatiuk", email = "aders1234@gmail.com" }] readme = "README.md" requires-python = ">= 3.11, < 3.14" dependencies = ["finecode_extension_api==0.3.*", "fine_python_ast==0.2.*"] + +[dependency-groups] +dev_workspace = ["finecode==0.3.*", "finecode_dev_common_preset==0.2.*"] + +[tool.finecode] +presets = [{ source = "finecode_dev_common_preset" }] + +[tool.finecode.env.dev_workspace.dependencies] +finecode_dev_common_preset = { path = "../../finecode_dev_common_preset", editable = true } +finecode = { path = "../../", editable = true } +finecode_extension_runner = { path = "../../finecode_extension_runner", editable = true } +finecode_extension_api = { path = "../../finecode_extension_api", editable = true } diff --git a/extensions/fine_python_mypy/pyproject.toml b/extensions/fine_python_mypy/pyproject.toml index 19762d0..5afecde 100644 --- a/extensions/fine_python_mypy/pyproject.toml +++ b/extensions/fine_python_mypy/pyproject.toml @@ -6,3 +6,15 @@ authors = [{ name = "Vladyslav Hnatiuk", email = "aders1234@gmail.com" }] readme = "README.md" requires-python = ">=3.11, < 3.14" dependencies = ["finecode_extension_api==0.3.*", "mypy (>=1.15, <2.0)"] + +[dependency-groups] +dev_workspace = ["finecode==0.3.*", "finecode_dev_common_preset==0.2.*"] + +[tool.finecode] +presets = [{ source = "finecode_dev_common_preset" }] + +[tool.finecode.env.dev_workspace.dependencies] +finecode_dev_common_preset = { path = "../../finecode_dev_common_preset", editable = true } +finecode = { path = "../../", editable = true } +finecode_extension_runner = { path = "../../finecode_extension_runner", editable = true } +finecode_extension_api = { path = "../../finecode_extension_api", editable = true } diff --git a/extensions/fine_python_package_info/fine_python_package_info/py_package_layout_info_provider.py b/extensions/fine_python_package_info/fine_python_package_info/py_package_layout_info_provider.py index c74584a..fe749fa 100644 --- a/extensions/fine_python_package_info/fine_python_package_info/py_package_layout_info_provider.py +++ b/extensions/fine_python_package_info/fine_python_package_info/py_package_layout_info_provider.py @@ -1,7 +1,7 @@ -import enum import pathlib import tomlkit +import tomlkit.exceptions from finecode_extension_api.interfaces import ifilemanager, ipypackagelayoutinfoprovider, icache from finecode_extension_api import service @@ -36,7 +36,7 @@ async def _get_package_name(self, package_dir_path: pathlib.Path) -> str: try: package_def_dict = tomlkit.loads(package_def_file_content) except tomlkit.exceptions.ParseError as exception: - raise ConfigParseError(f"Failed to parse package config {package_def_file}: {exception.message} at {exception.line}:{exception.col}") + raise ConfigParseError(f"Failed to parse package config {package_def_file}: toml parsing failed at {exception.line}:{exception.col}") from exception package_raw_name = package_def_dict.get('project', {}).get('name', None) if package_raw_name is None: diff --git a/extensions/fine_python_package_info/pyproject.toml b/extensions/fine_python_package_info/pyproject.toml index 557f467..bc18663 100644 --- a/extensions/fine_python_package_info/pyproject.toml +++ b/extensions/fine_python_package_info/pyproject.toml @@ -6,3 +6,15 @@ authors = [{ name = "Vladyslav Hnatiuk", email = "aders1234@gmail.com" }] readme = "README.md" requires-python = ">=3.11, < 3.14" dependencies = ["finecode_extension_api==0.3.*", "tomlkit==0.11.*"] + +[dependency-groups] +dev_workspace = ["finecode==0.3.*", "finecode_dev_common_preset==0.2.*"] + +[tool.finecode] +presets = [{ source = "finecode_dev_common_preset" }] + +[tool.finecode.env.dev_workspace.dependencies] +finecode_dev_common_preset = { path = "../../finecode_dev_common_preset", editable = true } +finecode = { path = "../../", editable = true } +finecode_extension_runner = { path = "../../finecode_extension_runner", editable = true } +finecode_extension_api = { path = "../../finecode_extension_api", editable = true } diff --git a/extensions/fine_python_pyrefly/fine_python_pyrefly/lint_handler.py b/extensions/fine_python_pyrefly/fine_python_pyrefly/lint_handler.py index 111ff93..08893fc 100644 --- a/extensions/fine_python_pyrefly/fine_python_pyrefly/lint_handler.py +++ b/extensions/fine_python_pyrefly/fine_python_pyrefly/lint_handler.py @@ -97,13 +97,20 @@ async def run_pyrefly_lint_on_single_file( venv_dir_path = self.extension_runner_info_provider.get_venv_dir_path_of_env(env_name=file_env) site_package_pathes = self.extension_runner_info_provider.get_venv_site_packages(venv_dir_path=venv_dir_path) + interpreter_path = self.extension_runner_info_provider.get_venv_python_interpreter(venv_dir_path=venv_dir_path) + # --skip-interpreter-query isn't used because it is not compatible + # with --python-interpreter parameter + # --disable-search-path-heuristics=true isn't used because pyrefly doesn't + # recognize some imports without it. For example, it cannot resolve relative + # imports in root __init__.py . Needs to be investigated cmd = [ str(self.pyrefly_bin_path), "check", "--output-format=json", - "--disable-search-path-heuristics=true", - "--skip-interpreter-query", + # path to python interpreter because pyrefly resolves .pth files only if + # it is provided + f"--python-interpreter='{str(interpreter_path)}'" ] if self.config.python_version is not None: diff --git a/extensions/fine_python_pyrefly/pyproject.toml b/extensions/fine_python_pyrefly/pyproject.toml index 2ac97b6..0fa9919 100644 --- a/extensions/fine_python_pyrefly/pyproject.toml +++ b/extensions/fine_python_pyrefly/pyproject.toml @@ -6,3 +6,15 @@ authors = [{ name = "Vladyslav Hnatiuk", email = "aders1234@gmail.com" }] readme = "README.md" requires-python = ">=3.11, < 3.14" dependencies = ["finecode_extension_api==0.3.*", "pyrefly (>=0.30.0,<1.0.0)"] + +[dependency-groups] +dev_workspace = ["finecode==0.3.*", "finecode_dev_common_preset==0.2.*"] + +[tool.finecode] +presets = [{ source = "finecode_dev_common_preset" }] + +[tool.finecode.env.dev_workspace.dependencies] +finecode_dev_common_preset = { path = "../../finecode_dev_common_preset", editable = true } +finecode = { path = "../../", editable = true } +finecode_extension_runner = { path = "../../finecode_extension_runner", editable = true } +finecode_extension_api = { path = "../../finecode_extension_api", editable = true } diff --git a/extensions/fine_python_ruff/fine_python_ruff/format_handler.py b/extensions/fine_python_ruff/fine_python_ruff/format_handler.py index f3ef0d7..44d5ed6 100644 --- a/extensions/fine_python_ruff/fine_python_ruff/format_handler.py +++ b/extensions/fine_python_ruff/fine_python_ruff/format_handler.py @@ -1,3 +1,5 @@ +# note: ruff formatter cannot sort imports, only ruff linter with fixes: +# https://docs.astral.sh/ruff/formatter/#sorting-imports from __future__ import annotations import dataclasses diff --git a/extensions/fine_python_ruff/fine_python_ruff/lint_handler.py b/extensions/fine_python_ruff/fine_python_ruff/lint_handler.py index 8dd414f..66e726c 100644 --- a/extensions/fine_python_ruff/fine_python_ruff/lint_handler.py +++ b/extensions/fine_python_ruff/fine_python_ruff/lint_handler.py @@ -16,6 +16,7 @@ class RuffLintHandlerConfig(code_action.ActionHandlerConfig): target_version: str = "py38" select: list[str] | None = None # Rules to enable ignore: list[str] | None = None # Rules to disable + extend_select: list[str] | None = None preview: bool = False @@ -98,11 +99,13 @@ async def run_ruff_lint_on_single_file( str(file_path), ] - if self.config.select: - cmd.extend(["--select", ",".join(self.config.select)]) - if self.config.ignore: - cmd.extend(["--ignore", ",".join(self.config.ignore)]) - if self.config.preview: + if self.config.select is not None: + cmd.append("--select=" + ",".join(self.config.select)) + if self.config.extend_select is not None: + cmd.append("--extend-select=" + ",".join(self.config.extend_select)) + if self.config.ignore is not None: + cmd.append("--ignore=" + ",".join(self.config.ignore)) + if self.config.preview is True: cmd.append("--preview") cmd_str = " ".join(cmd) diff --git a/extensions/fine_python_ruff/pyproject.toml b/extensions/fine_python_ruff/pyproject.toml index 63a9311..51525a4 100644 --- a/extensions/fine_python_ruff/pyproject.toml +++ b/extensions/fine_python_ruff/pyproject.toml @@ -6,3 +6,15 @@ authors = [{ name = "Vladyslav Hnatiuk", email = "aders1234@gmail.com" }] readme = "README.md" requires-python = ">=3.11, < 3.14" dependencies = ["finecode_extension_api==0.3.*", "ruff (>=0.8.0,<1.0.0)"] + +[dependency-groups] +dev_workspace = ["finecode==0.3.*", "finecode_dev_common_preset==0.2.*"] + +[tool.finecode] +presets = [{ source = "finecode_dev_common_preset" }] + +[tool.finecode.env.dev_workspace.dependencies] +finecode_dev_common_preset = { path = "../../finecode_dev_common_preset", editable = true } +finecode = { path = "../../", editable = true } +finecode_extension_runner = { path = "../../finecode_extension_runner", editable = true } +finecode_extension_api = { path = "../../finecode_extension_api", editable = true } diff --git a/extensions/fine_python_virtualenv/pyproject.toml b/extensions/fine_python_virtualenv/pyproject.toml index c8c3ab1..248ff45 100644 --- a/extensions/fine_python_virtualenv/pyproject.toml +++ b/extensions/fine_python_virtualenv/pyproject.toml @@ -20,3 +20,4 @@ presets = [{ source = "finecode_dev_common_preset" }] finecode_dev_common_preset = { path = "../../finecode_dev_common_preset", editable = true } finecode = { path = "../../", editable = true } finecode_extension_runner = { path = "../../finecode_extension_runner", editable = true } +finecode_extension_api = { path = "../../finecode_extension_api", editable = true } diff --git a/finecode_builtin_handlers/pyproject.toml b/finecode_builtin_handlers/pyproject.toml index 3453288..65587fd 100644 --- a/finecode_builtin_handlers/pyproject.toml +++ b/finecode_builtin_handlers/pyproject.toml @@ -14,6 +14,7 @@ dev_workspace = ["finecode==0.3.*", "finecode_dev_common_preset==0.2.*"] finecode_dev_common_preset = { path = "../finecode_dev_common_preset", editable = true } finecode = { path = "../", editable = true } finecode_extension_runner = { path = "../finecode_extension_runner", editable = true } +finecode_extension_api = { path = "../finecode_extension_api", editable = true } [tool.finecode] presets = [{ source = "finecode_dev_common_preset" }] diff --git a/finecode_extension_api/pyproject.toml b/finecode_extension_api/pyproject.toml index 79aa60f..db9bf87 100644 --- a/finecode_extension_api/pyproject.toml +++ b/finecode_extension_api/pyproject.toml @@ -16,4 +16,5 @@ presets = [{ source = "finecode_dev_common_preset" }] [tool.finecode.env.dev_workspace.dependencies] finecode_dev_common_preset = { path = "../finecode_dev_common_preset", editable = true } finecode = { path = "../", editable = true } +finecode_extension_api = { path = "./", editable = true } finecode_extension_runner = { path = "../finecode_extension_runner", editable = true } diff --git a/finecode_extension_api/src/finecode_extension_api/interfaces/iextensionrunnerinfoprovider.py b/finecode_extension_api/src/finecode_extension_api/interfaces/iextensionrunnerinfoprovider.py index 61f45dd..7bed5e3 100644 --- a/finecode_extension_api/src/finecode_extension_api/interfaces/iextensionrunnerinfoprovider.py +++ b/finecode_extension_api/src/finecode_extension_api/interfaces/iextensionrunnerinfoprovider.py @@ -10,3 +10,7 @@ def get_venv_dir_path_of_env(self, env_name: str) -> pathlib.Path: ... def get_venv_site_packages( self, venv_dir_path: pathlib.Path ) -> list[pathlib.Path]: ... + + def get_venv_python_interpreter( + self, venv_dir_path: pathlib.Path + ) -> pathlib.Path: ... diff --git a/finecode_extension_runner/pyproject.toml b/finecode_extension_runner/pyproject.toml index fce3a02..8cc33cb 100644 --- a/finecode_extension_runner/pyproject.toml +++ b/finecode_extension_runner/pyproject.toml @@ -28,6 +28,7 @@ dev = [{ include-group = "runtime" }, "pytest==7.4.*", "debugpy==1.8.*"] finecode_dev_common_preset = { path = "../finecode_dev_common_preset", editable = true } finecode = { path = "../", editable = true } finecode_extension_runner = { path = "../finecode_extension_runner", editable = true } +finecode_extension_api = { path = "../finecode_extension_api", editable = true } [build-system] requires = ["setuptools>=64", "setuptools-scm>=8"] diff --git a/finecode_extension_runner/src/finecode_extension_runner/_services/run_action.py b/finecode_extension_runner/src/finecode_extension_runner/_services/run_action.py index ac34f2e..34f285e 100644 --- a/finecode_extension_runner/src/finecode_extension_runner/_services/run_action.py +++ b/finecode_extension_runner/src/finecode_extension_runner/_services/run_action.py @@ -52,7 +52,9 @@ async def run_action( global last_run_id run_id = last_run_id last_run_id += 1 - logger.trace(f"Run action '{request.action_name}', run id: {run_id}, partial result token: {options.partial_result_token}") + logger.trace( + f"Run action '{request.action_name}', run id: {run_id}, partial result token: {options.partial_result_token}" + ) # TODO: check whether config is set: this will be solved by passing initial # configuration as payload of initialize if global_state.runner_context is None: @@ -558,9 +560,13 @@ async def run_subresult_coros_concurrently( action_subresult_type = type(coro_result) # use pydantic dataclass as constructor because it instantiates classes # recursively, normal dataclass only on the first level - action_subresult_type_pydantic = pydantic_dataclass(action_subresult_type) + action_subresult_type_pydantic = pydantic_dataclass( + action_subresult_type + ) action_subresult_dict = dataclasses.asdict(coro_result) - action_subresult = action_subresult_type_pydantic(**action_subresult_dict) + action_subresult = action_subresult_type_pydantic( + **action_subresult_dict + ) else: action_subresult.update(coro_result) diff --git a/finecode_extension_runner/src/finecode_extension_runner/cli.py b/finecode_extension_runner/src/finecode_extension_runner/cli.py index 2e78ddb..73aea9f 100644 --- a/finecode_extension_runner/src/finecode_extension_runner/cli.py +++ b/finecode_extension_runner/src/finecode_extension_runner/cli.py @@ -48,10 +48,7 @@ def start( global_state.log_level = "INFO" if trace is False else "TRACE" global_state.project_dir_path = project_path global_state.env_name = env_name - # asyncio.run(runner_start.start_runner()) - # extension runner doesn't stop with async start after closing LS client(WM). Use - # sync start until this problem is solved runner_start.start_runner_sync(env_name) diff --git a/finecode_extension_runner/src/finecode_extension_runner/impls/extension_runner_info_provider.py b/finecode_extension_runner/src/finecode_extension_runner/impls/extension_runner_info_provider.py index 281da0b..236a5d6 100644 --- a/finecode_extension_runner/src/finecode_extension_runner/impls/extension_runner_info_provider.py +++ b/finecode_extension_runner/src/finecode_extension_runner/impls/extension_runner_info_provider.py @@ -55,3 +55,8 @@ def get_venv_site_packages(self, venv_dir_path: pathlib.Path) -> list[pathlib.Pa self._site_packages_cache[venv_dir_path] = site_packages return site_packages + + def get_venv_python_interpreter(self, venv_dir_path: pathlib.Path) -> pathlib.Path: + bin_dir_path = venv_dir_path / "bin" + interpreter_exe = bin_dir_path / "python" + return interpreter_exe diff --git a/finecode_extension_runner/src/finecode_extension_runner/impls/project_file_classifier.py b/finecode_extension_runner/src/finecode_extension_runner/impls/project_file_classifier.py index b444f84..1477c7b 100644 --- a/finecode_extension_runner/src/finecode_extension_runner/impls/project_file_classifier.py +++ b/finecode_extension_runner/src/finecode_extension_runner/impls/project_file_classifier.py @@ -1,4 +1,3 @@ -import enum import pathlib from finecode_extension_api.interfaces import ( diff --git a/finecode_extension_runner/src/finecode_extension_runner/impls/project_info_provider.py b/finecode_extension_runner/src/finecode_extension_runner/impls/project_info_provider.py index 4c8a469..5f2d06d 100644 --- a/finecode_extension_runner/src/finecode_extension_runner/impls/project_info_provider.py +++ b/finecode_extension_runner/src/finecode_extension_runner/impls/project_info_provider.py @@ -24,7 +24,9 @@ async def get_current_project_package_name(self) -> str: project_raw_config = await self.get_current_project_raw_config() raw_name = project_raw_config.get("project", {}).get("name", None) if raw_name is None: - raise iprojectinfoprovider.InvalidProjectConfig("project.name not found in project config") + raise iprojectinfoprovider.InvalidProjectConfig( + "project.name not found in project config" + ) return raw_name.replace("-", "_") diff --git a/finecode_extension_runner/src/finecode_extension_runner/logs.py b/finecode_extension_runner/src/finecode_extension_runner/logs.py index a9ea3dd..942bcdb 100644 --- a/finecode_extension_runner/src/finecode_extension_runner/logs.py +++ b/finecode_extension_runner/src/finecode_extension_runner/logs.py @@ -39,7 +39,7 @@ def save_logs_to_file( file_path: Path, log_level: str = "INFO", rotation: str = "10 MB", - retention=3, + retention: int = 3, stdout: bool = True, ): if stdout is True: diff --git a/finecode_extension_runner/src/finecode_extension_runner/lsp_server.py b/finecode_extension_runner/src/finecode_extension_runner/lsp_server.py index 33b6edd..96be19c 100644 --- a/finecode_extension_runner/src/finecode_extension_runner/lsp_server.py +++ b/finecode_extension_runner/src/finecode_extension_runner/lsp_server.py @@ -10,18 +10,82 @@ import functools import json import pathlib -import time import typing import pygls.exceptions as pygls_exceptions from loguru import logger from lsprotocol import types from pygls.lsp import server as lsp_server +from pygls.io_ import StdoutWriter, run_async from finecode_extension_api import code_action from finecode_extension_runner import domain, schemas, services from finecode_extension_runner._services import run_action as run_action_service +import sys +import io +import threading +import asyncio + + +class StdinAsyncReader: + """Read from stdin asynchronously.""" + + def __init__(self, stdin: io.TextIO, stop_event: threading.Event | None = None): + self.stdin = stdin + self._loop: asyncio.AbstractEventLoop | None = None + self._stop_event = stop_event + + self.reader = asyncio.StreamReader() + self.transport: asyncio.ReadTransport | None = None + self.initialized = False + + @property + def loop(self): + if self._loop is None: + self._loop = asyncio.get_running_loop() + + return self._loop + + async def readline(self) -> bytes: + if not self.initialized: + await self.initialize() + + while not self._stop_event.is_set(): + try: + line = await asyncio.wait_for(self.reader.readline(), timeout=0.1) + if not line: # EOF + break + return line + except TimeoutError: + ... + return bytes() + + async def readexactly(self, n: int) -> bytes: + if not self.initialized: + await self.initialize() + + while not self._stop_event.is_set(): + try: + line = await asyncio.wait_for(self.reader.read(n), timeout=0.1) + if not line: # EOF + break + return line + except TimeoutError: + ... + return bytes() + + async def initialize(self) -> None: + protocol = asyncio.StreamReaderProtocol(self.reader) + self.transport, _ = await self.loop.connect_read_pipe( + lambda: protocol, self.stdin + ) + self.initialized = True + + def stop(self) -> None: + if self.transport: + self.transport.close() + class CustomLanguageServer(lsp_server.LanguageServer): def report_server_error(self, error: Exception, source: lsp_server.ServerErrors): @@ -30,6 +94,35 @@ def report_server_error(self, error: Exception, source: lsp_server.ServerErrors) # send to client super().report_server_error(error, source) + async def start_io_async( + self, stdin: io.BinaryIO | None = None, stdout: io.BinaryIO | None = None + ): + """Starts an asynchronous IO server.""" + # overwrite this method to use custom StdinAsyncReader which handles stop event properly + logger.info("Starting async IO server") + + self._stop_event = threading.Event() + reader = StdinAsyncReader(sys.stdin, self._stop_event) + writer = StdoutWriter(stdout or sys.stdout.buffer) + self.protocol.set_writer(writer) + + try: + await run_async( + stop_event=self._stop_event, + reader=reader, + protocol=self.protocol, + logger=logger, + error_handler=self.report_server_error, + ) + except BrokenPipeError: + logger.error("Connection to the client is lost! Shutting down the server.") + except (KeyboardInterrupt, SystemExit): + logger.info("exception handler in json rpc server") + pass + finally: + reader.stop() + self.shutdown() + def create_lsp_server() -> lsp_server.LanguageServer: server = CustomLanguageServer("FineCode_Extension_Runner_Server", "v1") @@ -40,6 +133,9 @@ def create_lsp_server() -> lsp_server.LanguageServer: register_shutdown_feature = server.feature(types.SHUTDOWN) register_shutdown_feature(_on_shutdown) + register_exit_feature = server.feature(types.EXIT) + register_exit_feature(_on_exit) + register_document_did_open_feature = server.feature(types.TEXT_DOCUMENT_DID_OPEN) register_document_did_open_feature(_document_did_open) @@ -61,8 +157,6 @@ def create_lsp_server() -> lsp_server.LanguageServer: def on_process_exit(): logger.info("Exit extension runner") services.shutdown_all_action_handlers() - # wait for graceful shutdown of all subprocesses if such exist - time.sleep(2) services.exit_all_action_handlers() atexit.register(on_process_exit) @@ -72,7 +166,9 @@ def send_partial_result( ) -> None: partial_result_dict = dataclasses.asdict(partial_result) partial_result_json = json.dumps(partial_result_dict) - logger.debug(f"Send partial result for {token}, length {len(partial_result_json)}") + logger.debug( + f"Send partial result for {token}, length {len(partial_result_json)}" + ) server.progress(types.ProgressParams(token=token, value=partial_result_json)) run_action_service.set_partial_result_sender(send_partial_result) @@ -87,6 +183,12 @@ def _on_initialized(ls: lsp_server.LanguageServer, params: types.InitializedPara def _on_shutdown(ls: lsp_server.LanguageServer, params): logger.info("Shutdown extension runner") services.shutdown_all_action_handlers() + logger.info("Shutdown end") + return None + + +def _on_exit(ls: lsp_server.LanguageServer, params): + logger.info("Exit extension runner") def _document_did_open( @@ -105,9 +207,11 @@ def _document_did_close( async def document_requester(server: lsp_server.LanguageServer, uri: str): try: - document = await server.protocol.send_request_async( - "documents/get", params={"uri": uri} + document = await asyncio.wait_for( + server.protocol.send_request_async("documents/get", params={"uri": uri}), 10 ) + except TimeoutError as error: + raise error except pygls_exceptions.JsonRpcInternalError as error: if error.message == "Exception: Document is not opened": raise domain.TextDocumentNotOpened() @@ -120,9 +224,13 @@ async def document_requester(server: lsp_server.LanguageServer, uri: str): async def document_saver(server: lsp_server.LanguageServer, uri: str, content: str): - document = await server.protocol.send_request_async( - "documents/get", params={"uri": uri} - ) + try: + document = await asyncio.wait_for( + server.protocol.send_request_async("documents/get", params={"uri": uri}), 10 + ) + except TimeoutError as error: + raise error + document_lines = document.text.split("\n") params = types.ApplyWorkspaceEditParams( edit=types.WorkspaceEdit( @@ -156,9 +264,14 @@ async def get_project_raw_config( server: lsp_server.LanguageServer, project_def_path: str ) -> dict[str, typing.Any]: try: - raw_config = await server.protocol.send_request_async( - "projects/getRawConfig", params={"projectDefPath": project_def_path} + raw_config = await asyncio.wait_for( + server.protocol.send_request_async( + "projects/getRawConfig", params={"projectDefPath": project_def_path} + ), + 10, ) + except TimeoutError as error: + raise error except pygls_exceptions.JsonRpcInternalError as error: raise error diff --git a/finecode_extension_runner/src/finecode_extension_runner/services.py b/finecode_extension_runner/src/finecode_extension_runner/services.py index 415338a..46d34b5 100644 --- a/finecode_extension_runner/src/finecode_extension_runner/services.py +++ b/finecode_extension_runner/src/finecode_extension_runner/services.py @@ -185,13 +185,18 @@ def shutdown_action_handler( def shutdown_all_action_handlers() -> None: - logger.trace("Shutdown all action handlers") - for action_cache in global_state.runner_context.action_cache_by_name.values(): - for handler_name, handler_cache in action_cache.handler_cache_by_name.items(): - if handler_cache.exec_info is not None: - shutdown_action_handler( - action_handler_name=handler_name, exec_info=handler_cache.exec_info - ) + if global_state.runner_context is not None: + logger.trace("Shutdown all action handlers") + for action_cache in global_state.runner_context.action_cache_by_name.values(): + for ( + handler_name, + handler_cache, + ) in action_cache.handler_cache_by_name.items(): + if handler_cache.exec_info is not None: + shutdown_action_handler( + action_handler_name=handler_name, + exec_info=handler_cache.exec_info, + ) def exit_action_handler( @@ -209,12 +214,16 @@ def exit_action_handler( def exit_all_action_handlers() -> None: - logger.trace("Exit all action handlers") - for action_cache in global_state.runner_context.action_cache_by_name.values(): - for handler_name, handler_cache in action_cache.handler_cache_by_name.items(): - if handler_cache.exec_info is not None: - exec_info = handler_cache.exec_info - exit_action_handler( - action_handler_name=handler_name, exec_info=exec_info - ) - action_cache.handler_cache_by_name = {} + if global_state.runner_context is not None: + logger.trace("Exit all action handlers") + for action_cache in global_state.runner_context.action_cache_by_name.values(): + for ( + handler_name, + handler_cache, + ) in action_cache.handler_cache_by_name.items(): + if handler_cache.exec_info is not None: + exec_info = handler_cache.exec_info + exit_action_handler( + action_handler_name=handler_name, exec_info=exec_info + ) + action_cache.handler_cache_by_name = {} diff --git a/finecode_extension_runner/src/finecode_extension_runner/start.py b/finecode_extension_runner/src/finecode_extension_runner/start.py index 904868a..e8b0f51 100644 --- a/finecode_extension_runner/src/finecode_extension_runner/start.py +++ b/finecode_extension_runner/src/finecode_extension_runner/start.py @@ -1,5 +1,6 @@ import inspect import logging +import socket import sys from loguru import logger @@ -8,56 +9,12 @@ import finecode_extension_runner.lsp_server as extension_runner_lsp from finecode_extension_runner import logs -# import finecode.pygls_server_utils as pygls_server_utils - - -# async def start_runner(): -# project_log_dir_path = project_dirs.get_project_dir(global_state.project_dir_path) -# logger.remove() -# # ~~extension runner communicates with workspace manager with tcp, we can print -# # logs to stdout as well~~. See README.md -# logs.save_logs_to_file( -# file_path=project_log_dir_path / "execution.log", -# log_level=global_state.log_level, -# stdout=False, -# ) - -# # pygls uses standard python logger, intercept it and pass logs to loguru -# class InterceptHandler(logging.Handler): -# def emit(self, record: logging.LogRecord) -> None: -# # Get corresponding Loguru level if it exists. -# level: str | int -# try: -# level = logger.level(record.levelname).name -# except ValueError: -# level = record.levelno - -# # Find caller from where originated the logged message. -# frame, depth = inspect.currentframe(), 0 -# while frame and ( -# depth == 0 or frame.f_code.co_filename == logging.__file__ -# ): -# frame = frame.f_back -# depth += 1 - -# logger.opt(depth=depth, exception=record.exc_info).log( -# level, record.getMessage() -# ) - -# logging.basicConfig(handlers=[InterceptHandler()], level=0, force=True) - -# logger.info(f"Python executable: {sys.executable}") -# logger.info(f"Project path: {global_state.project_dir_path}") - -# server = extension_runner_lsp.create_lsp_server() -# await pygls_server_utils.start_io_async(server) - def start_runner_sync(env_name: str) -> None: logger.remove() # disable logging raw messages # TODO: make configurable - logger.configure(activation=[("pygls.protocol.json_rpc", False)]) + # logger.configure(activation=[("pygls.protocol.json_rpc", False)]) # ~~extension runner communicates with workspace manager with tcp, we can print logs # to stdout as well~~. See README.md assert global_state.project_dir_path is not None @@ -68,7 +25,7 @@ def start_runner_sync(env_name: str) -> None: / "logs" / "runner.log", log_level=global_state.log_level, - stdout=False, + stdout=True, ) # pygls uses standard python logger, intercept it and pass logs to loguru @@ -107,4 +64,15 @@ def emit(self, record: logging.LogRecord) -> None: logger.info(f"Project path: {global_state.project_dir_path}") server = extension_runner_lsp.create_lsp_server() - server.start_io() + # asyncio.run(server.start_io_async()) + port = _find_free_port() + server.start_tcp(host="127.0.0.1", port=port) + + +def _find_free_port() -> int: + """Find and return a free TCP port.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("", 0)) + s.listen(1) + port = s.getsockname()[1] + return port diff --git a/presets/fine_python_lint/fine_python_lint/preset.toml b/presets/fine_python_lint/fine_python_lint/preset.toml index dc81393..2bf41e3 100644 --- a/presets/fine_python_lint/fine_python_lint/preset.toml +++ b/presets/fine_python_lint/fine_python_lint/preset.toml @@ -12,6 +12,10 @@ handlers = [ ] }, ] +[[tool.finecode.action_handler]] +source = "fine_python_ruff.RuffLintHandler" +config.extend_select = ["B", "I"] + # flake8 is used only for custom rules, all standard rules are checked by ruff, but # keep flake8 configuration if someone activates some rules or uses flake8 config # parameters in their own rules @@ -24,3 +28,14 @@ config.max_line_length = 80 config.extend_ignore = ["E203", "E501", "E701", "W391"] # disable all standard rules config.select = [] + +[[tool.finecode.action_handler]] +source = "fine_python_pip.PipInstallDepsInEnvHandler" +# use compat editable mode because some static analysis tools like pyrefly support +# only .pth files with pathes, not with executable code lines: +# https://pyrefly.org/en/docs/import-resolution/#editable-installs +# another possibility is strict mode, but it doesn't work in this case, because it +# creates editable directory directly in the package and if the package is installed +# as editable in multiple packages, there are concurrency problems and it would require +# sequential installation of dependencies in all envs. +config.editable_mode = 'compat' diff --git a/pyproject.toml b/pyproject.toml index 4f7fa72..c2155f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ dynamic = ["version"] description = "" authors = [{ name = "Vladyslav Hnatiuk", email = "aders1234@gmail.com" }] readme = "README.md" -requires-python = ">=3.11, < 3.14" +requires-python = ">=3.11, <= 3.14" dependencies = [ "loguru==0.7.*", "tomlkit==0.11.*", @@ -19,6 +19,8 @@ dependencies = [ "mcp==1.13.*", "fine_python_virtualenv==0.1.*", "fine_python_pip==0.1.*", + "culsans==0.9.*", + "apischema==0.19.*", ] [dependency-groups] @@ -44,6 +46,7 @@ presets = [{ source = "finecode_dev_common_preset" }] [tool.finecode.env.dev_workspace.dependencies] finecode_dev_common_preset = { path = "./finecode_dev_common_preset", editable = true } finecode_extension_runner = { path = "./finecode_extension_runner", editable = true } +finecode_extension_api = { path = "./finecode_extension_api", editable = true } [tool.importlinter] root_package = "finecode" @@ -79,3 +82,6 @@ version_file = "src/finecode/_version.py" [tool.setuptools.package-data] finecode = ["base_config.toml"] + +[tool.pyright] +reportUnusedCallResult = false diff --git a/src/finecode/cli.py b/src/finecode/cli.py index 2385ffa..08f081b 100644 --- a/src/finecode/cli.py +++ b/src/finecode/cli.py @@ -8,7 +8,7 @@ import click from loguru import logger -import finecode.main as workspace_manager +import finecode.lsp_server.main as wm_lsp_server from finecode import communication_utils, logger_utils, user_messages from finecode.cli_app.commands import dump_config_cmd, prepare_envs_cmd, run_cmd @@ -66,17 +66,9 @@ def start_api( else: raise ValueError("Specify either --tcp, --ws or --stdio") - # loop = asyncio.get_event_loop() - # loop.run_until_complete( - # workspace_manager.start( - # comm_type=comm_type, host=host, port=port, trace=trace - # ) - # ) - # loop.run_forever() - - # workspace manager doesn't stop with async start after closing LS client(IDE). - # Use sync start until this problem is solved - workspace_manager.start_sync(comm_type=comm_type, host=host, port=port, trace=trace) + asyncio.run( + wm_lsp_server.start(comm_type=comm_type, host=host, port=port, trace=trace) + ) async def show_user_message(message: str, message_type: str) -> None: diff --git a/src/finecode/cli_app/commands/dump_config_cmd.py b/src/finecode/cli_app/commands/dump_config_cmd.py index 281b2a7..93fb593 100644 --- a/src/finecode/cli_app/commands/dump_config_cmd.py +++ b/src/finecode/cli_app/commands/dump_config_cmd.py @@ -5,7 +5,7 @@ from finecode import context from finecode.services import run_service, shutdown_service from finecode.config import config_models, read_configs -from finecode.runner import manager as runner_manager +from finecode.runner import runner_manager class DumpFailed(Exception): diff --git a/src/finecode/cli_app/commands/prepare_envs_cmd.py b/src/finecode/cli_app/commands/prepare_envs_cmd.py index 87d8f7b..bf749c4 100644 --- a/src/finecode/cli_app/commands/prepare_envs_cmd.py +++ b/src/finecode/cli_app/commands/prepare_envs_cmd.py @@ -7,7 +7,7 @@ from finecode.services import run_service, shutdown_service from finecode.cli_app import utils from finecode.config import collect_actions, config_models, read_configs -from finecode.runner import manager as runner_manager +from finecode.runner import runner_manager class PrepareEnvsFailed(Exception): ... diff --git a/src/finecode/cli_app/commands/run_cmd.py b/src/finecode/cli_app/commands/run_cmd.py index e17912b..de205fd 100644 --- a/src/finecode/cli_app/commands/run_cmd.py +++ b/src/finecode/cli_app/commands/run_cmd.py @@ -1,15 +1,12 @@ -import asyncio import pathlib -from typing import NamedTuple -import click import ordered_set from loguru import logger from finecode import context, domain from finecode.services import run_service, shutdown_service from finecode.config import collect_actions, config_models, read_configs -from finecode.runner import manager as runner_manager +from finecode.runner import runner_manager from finecode.cli_app import utils diff --git a/src/finecode/config/read_configs.py b/src/finecode/config/read_configs.py index c288948..5ba16de 100644 --- a/src/finecode/config/read_configs.py +++ b/src/finecode/config/read_configs.py @@ -7,7 +7,7 @@ from finecode import context, domain, user_messages from finecode.config import config_models -from finecode.runner import runner_client, runner_info +from finecode.runner import runner_client async def read_projects_in_dir( @@ -40,7 +40,7 @@ async def read_projects_in_dir( actions: list[domain.Action] | None = None with open(def_file, "rb") as pyproject_file: - project_def = toml_loads(pyproject_file.read()).value + project_def = toml_loads(pyproject_file.read()).unwrap() dependency_groups = project_def.get("dependency-groups", {}) dev_workspace_group = dependency_groups.get("dev_workspace", []) @@ -112,13 +112,13 @@ async def read_project_config( if project.def_path.name == "pyproject.toml": with open(project.def_path, "rb") as pyproject_file: # TODO: handle error if toml is invalid - project_def = toml_loads(pyproject_file.read()).value + project_def = toml_loads(pyproject_file.read()).unwrap() # TODO: validate that finecode is installed? base_config_path = Path(__file__).parent.parent / "base_config.toml" # TODO: cache instead of reading each time with open(base_config_path, "r") as base_config_file: - base_config = toml_loads(base_config_file.read()).value + base_config = toml_loads(base_config_file.read()).unwrap() project_config = {} _merge_projects_configs( project_config, project.def_path, base_config, base_config_path @@ -188,7 +188,7 @@ class PresetToProcess(NamedTuple): async def get_preset_project_path( - preset: PresetToProcess, def_path: Path, runner: runner_info.ExtensionRunnerInfo + preset: PresetToProcess, def_path: Path, runner: runner_client.ExtensionRunnerInfo ) -> Path | None: logger.trace(f"Get preset project path: {preset.source}") @@ -231,7 +231,7 @@ def read_preset_config( ) with open(config_path, "rb") as preset_toml_file: - preset_toml = toml_loads(preset_toml_file.read()).value + preset_toml = toml_loads(preset_toml_file.read()).unwrap() try: presets = preset_toml["tool"]["finecode"]["presets"] @@ -245,7 +245,9 @@ def read_preset_config( async def collect_config_from_py_presets( - presets_sources: list[str], def_path: Path, runner: runner_info.ExtensionRunnerInfo + presets_sources: list[str], + def_path: Path, + runner: runner_client.ExtensionRunnerInfo, ) -> dict[str, Any] | None: config: dict[str, Any] | None = None processed_presets: set[str] = set() @@ -528,7 +530,7 @@ def handler_to_dict(handler: domain.ActionHandler) -> dict[str, str | list[str]] def add_runtime_dependency_group_if_new(project_config: dict[str, Any]) -> None: runtime_dependencies = project_config.get("project", {}).get("dependencies", []) - + # add root package to runtime env if it is not there yet. It is done here and not # in package installer, because runtime deps group can be included in other groups # and root package should be installed in them as well @@ -536,31 +538,33 @@ def add_runtime_dependency_group_if_new(project_config: dict[str, Any]) -> None: if root_package_name is None: raise config_models.ConfigurationError("project.name not found in config") root_package_in_runtime_deps = any( - dep for dep in runtime_dependencies if get_dependency_name(dep) == root_package_name + dep + for dep in runtime_dependencies + if get_dependency_name(dep) == root_package_name ) if not root_package_in_runtime_deps: runtime_dependencies.insert(0, root_package_name) - + # make editable. Example: # [tool.finecode.env.runtime.dependencies] # package_name = { path = "./", editable = true } - if 'tool' not in project_config: - project_config['tool'] = {} - tool_config = project_config['tool'] - if 'finecode' not in tool_config: - tool_config['finecode'] = {} - finecode_config = tool_config['finecode'] - if 'env' not in finecode_config: - finecode_config['env'] = {} - finecode_env_config = finecode_config['env'] - if 'runtime' not in finecode_env_config: - finecode_env_config['runtime'] = {} - runtime_env_config = finecode_env_config['runtime'] - if 'dependencies' not in runtime_env_config: - runtime_env_config['dependencies'] = {} - runtime_env_deps = runtime_env_config['dependencies'] + if "tool" not in project_config: + project_config["tool"] = {} + tool_config = project_config["tool"] + if "finecode" not in tool_config: + tool_config["finecode"] = {} + finecode_config = tool_config["finecode"] + if "env" not in finecode_config: + finecode_config["env"] = {} + finecode_env_config = finecode_config["env"] + if "runtime" not in finecode_env_config: + finecode_env_config["runtime"] = {} + runtime_env_config = finecode_env_config["runtime"] + if "dependencies" not in runtime_env_config: + runtime_env_config["dependencies"] = {} + runtime_env_deps = runtime_env_config["dependencies"] if root_package_name not in runtime_env_deps: - runtime_env_deps[root_package_name] = { "path": "./", "editable": True} + runtime_env_deps[root_package_name] = {"path": "./", "editable": True} deps_groups = add_or_get_dict_key_value(project_config, "dependency-groups", {}) if "runtime" not in deps_groups: diff --git a/src/finecode/context.py b/src/finecode/context.py index ffbcca1..7bd0914 100644 --- a/src/finecode/context.py +++ b/src/finecode/context.py @@ -7,7 +7,8 @@ from finecode import domain if TYPE_CHECKING: - from finecode.runner.runner_info import ExtensionRunnerInfo + from finecode.runner.runner_client import ExtensionRunnerInfo + from finecode.runner._io_thread import AsyncIOThread @dataclass @@ -23,6 +24,7 @@ class WorkspaceContext: ws_projects_extension_runners: dict[Path, dict[str, ExtensionRunnerInfo]] = field( default_factory=dict ) + runner_io_thread: AsyncIOThread | None = None ignore_watch_paths: set[Path] = field(default_factory=set) # we save list of meta and pygls manages content of documents automatically. diff --git a/src/finecode/domain.py b/src/finecode/domain.py index 4229e93..436774a 100644 --- a/src/finecode/domain.py +++ b/src/finecode/domain.py @@ -27,6 +27,15 @@ def __init__( self.env: str = env self.dependencies: list[str] = dependencies + def to_dict(self) -> dict[str, typing.Any]: + return { + "name": self.name, + "source": self.source, + "config": self.config, + "env": self.env, + "dependencies": self.dependencies, + } + class Action: def __init__( @@ -41,6 +50,14 @@ def __init__( self.handlers: list[ActionHandler] = handlers self.config = config + def to_dict(self) -> dict[str, typing.Any]: + return { + "name": self.name, + "source": self.source, + "handlers": [handler.to_dict() for handler in self.handlers], + "config": self.config, + } + class Project: def __init__( diff --git a/src/finecode/find_project.py b/src/finecode/find_project.py index f0ba83d..508c720 100644 --- a/src/finecode/find_project.py +++ b/src/finecode/find_project.py @@ -4,7 +4,7 @@ from finecode import domain from finecode.context import WorkspaceContext -from finecode.runner import manager as runner_manager +from finecode.runner import runner_manager class FileNotInWorkspaceException(BaseException): ... diff --git a/src/finecode/logger_utils.py b/src/finecode/logger_utils.py index a9e0df3..4aedf7a 100644 --- a/src/finecode/logger_utils.py +++ b/src/finecode/logger_utils.py @@ -9,7 +9,7 @@ def init_logger(trace: bool, stdout: bool = False): - venv_dir_path = Path(sys.executable) / ".." / ".." + venv_dir_path = Path(sys.executable).parent.parent logs_dir_path = venv_dir_path / "logs" logger.remove() @@ -19,6 +19,7 @@ def init_logger(trace: bool, stdout: bool = False): activation=[ ("pygls.protocol.json_rpc", False), ("pygls.feature_manager", False), + ("pygls.io_", False), ] ) logs.save_logs_to_file( diff --git a/src/finecode/lsp_server/endpoints/diagnostics.py b/src/finecode/lsp_server/endpoints/diagnostics.py index cf01a47..b1e9f2d 100644 --- a/src/finecode/lsp_server/endpoints/diagnostics.py +++ b/src/finecode/lsp_server/endpoints/diagnostics.py @@ -15,7 +15,6 @@ domain, project_analyzer, pygls_types_utils, - services, ) from finecode.services import run_service from finecode.lsp_server import global_state @@ -128,7 +127,6 @@ async def document_diagnostic_with_partial_results( async with run_service.find_action_project_and_run_with_partial_results( file_path=file_path, action_name="lint", - # TODO: use payload class params={ "file_paths": [file_path], }, @@ -285,7 +283,7 @@ async def run_workspace_diagnostic_with_partial_results( async def workspace_diagnostic_with_partial_results( exec_infos: list[LintActionExecInfo], partial_result_token: str | int -) -> WorkspaceDiagnosticReport: +) -> types.WorkspaceDiagnosticReport: try: async with asyncio.TaskGroup() as tg: for exec_info in exec_infos: @@ -311,7 +309,7 @@ async def workspace_diagnostic_with_full_result( for exec_info in exec_infos: project = ws_context.ws_projects[exec_info.project_dir_path] task = tg.create_task( - services.run_action( + run_service.run_action( action_name=exec_info.action_name, params=exec_info.request_data, project_def=project, diff --git a/src/finecode/lsp_server/endpoints/document_sync.py b/src/finecode/lsp_server/endpoints/document_sync.py index 56fd933..d1b5081 100644 --- a/src/finecode/lsp_server/endpoints/document_sync.py +++ b/src/finecode/lsp_server/endpoints/document_sync.py @@ -7,7 +7,7 @@ from finecode import domain from finecode.lsp_server import global_state -from finecode.runner import runner_client, runner_info +from finecode.runner import runner_client async def document_did_open( @@ -40,11 +40,12 @@ async def document_did_open( ) ) for runner in runners_by_env.values(): - tg.create_task( - runner_client.notify_document_did_open( - runner=runner, document_info=document_info + if runner.status == runner_client.RunnerStatus.RUNNING: + tg.create_task( + runner_client.notify_document_did_open( + runner=runner, document_info=document_info + ) ) - ) except ExceptionGroup as eg: for exception in eg.exceptions: logger.exception(exception) @@ -78,7 +79,7 @@ async def document_did_close( project_path ] for runner in runners_by_env.values(): - if runner.status != runner_info.RunnerStatus.RUNNING: + if runner.status != runner_client.RunnerStatus.RUNNING: logger.trace( f"Runner {runner.readable_id} is not running, skip it" ) diff --git a/src/finecode/lsp_server/endpoints/formatting.py b/src/finecode/lsp_server/endpoints/formatting.py index 51d003c..e393e54 100644 --- a/src/finecode/lsp_server/endpoints/formatting.py +++ b/src/finecode/lsp_server/endpoints/formatting.py @@ -57,7 +57,7 @@ async def format_document(ls: LanguageServer, params: types.DocumentFormattingPa async def format_range(ls: LanguageServer, params: types.DocumentRangeFormattingParams): logger.info(f"format range {params}") await global_state.server_initialized.wait() - + # TODO return [] @@ -66,5 +66,5 @@ async def format_ranges( ): logger.info(f"format ranges {params}") await global_state.server_initialized.wait() - + # TODO return [] diff --git a/src/finecode/lsp_server/endpoints/inlay_hints.py b/src/finecode/lsp_server/endpoints/inlay_hints.py index a8a4cf5..0d20b66 100644 --- a/src/finecode/lsp_server/endpoints/inlay_hints.py +++ b/src/finecode/lsp_server/endpoints/inlay_hints.py @@ -73,4 +73,6 @@ async def document_inlay_hint( async def inlay_hint_resolve( ls: LanguageServer, params: types.InlayHint -) -> types.InlayHint | None: ... +) -> types.InlayHint | None: + # TODO + ... diff --git a/src/finecode/lsp_server/lsp_server.py b/src/finecode/lsp_server/lsp_server.py index 9d7c1b3..454e73f 100644 --- a/src/finecode/lsp_server/lsp_server.py +++ b/src/finecode/lsp_server/lsp_server.py @@ -6,9 +6,10 @@ from loguru import logger from lsprotocol import types from pygls.lsp.server import LanguageServer +from finecode_extension_runner.lsp_server import CustomLanguageServer from finecode.services import shutdown_service -from finecode.runner import manager as runner_manager +from finecode.runner import runner_manager, runner_client from finecode.lsp_server import global_state, schemas, services from finecode.lsp_server.endpoints import action_tree as action_tree_endpoints from finecode.lsp_server.endpoints import code_actions as code_actions_endpoints @@ -19,13 +20,14 @@ from finecode.lsp_server.endpoints import inlay_hints as inlay_hints_endpoints -def create_lsp_server() -> LanguageServer: +def create_lsp_server() -> CustomLanguageServer: # handle all requests explicitly because there are different types of requests: # project-specific, workspace-wide. Some Workspace-wide support partial responses, # some not. - server = LanguageServer( - "FineCode_Workspace_Manager_Server", "v1" - ) + # + # use CustomLanguageServer, because the problem with stopping the server with IO + # communication(stopping waiting on input) is solved in it + server = CustomLanguageServer("FineCode_Workspace_Manager_Server", "v1") register_initialized_feature = server.feature(types.INITIALIZED) register_initialized_feature(_on_initialized) @@ -164,9 +166,13 @@ def pass_log_to_ls_client(log) -> None: # loguru doesn't support passing partial with ls parameter, use nested function # instead - logger.add(sink=pass_log_to_ls_client) + # + # Disabled, because it is not thread-safe and it means not compatible with IO thread + # logger.add(sink=pass_log_to_ls_client) - async def get_document(params): + async def get_document( + params: runner_client.GetDocumentParams, + ) -> runner_client.GetDocumentResult: try: doc_info = global_state.ws_context.opened_documents[params.uri] except KeyError: @@ -183,7 +189,9 @@ async def get_document(params): raise Exception("Document is not opened") text = ls.workspace.get_text_document(params.uri).source - return {"uri": params.uri, "version": doc_info.version, "text": text} + return runner_client.GetDocumentResult( + uri=params.uri, version=doc_info.version, text=text + ) logger.info("initialized, adding workspace directories") @@ -264,7 +272,7 @@ async def restart_extension_runner(ls: LanguageServer, params): ) -def send_user_message_notification( +async def send_user_message_notification( ls: LanguageServer, message: str, message_type: str ) -> None: message_type_pascal = message_type[0] + message_type[1:].lower() diff --git a/src/finecode/lsp_server/main.py b/src/finecode/lsp_server/main.py new file mode 100644 index 0000000..c6f2e83 --- /dev/null +++ b/src/finecode/lsp_server/main.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +from finecode import communication_utils +from finecode import logger_utils +from finecode.lsp_server.lsp_server import create_lsp_server + + +async def start( + comm_type: communication_utils.CommunicationType, + host: str | None = None, + port: int | None = None, + trace: bool = False, +) -> None: + logger_utils.init_logger(trace=trace) + server = create_lsp_server() + await server.start_io_async() diff --git a/src/finecode/lsp_server/services.py b/src/finecode/lsp_server/services.py index 8533fa3..1af70db 100644 --- a/src/finecode/lsp_server/services.py +++ b/src/finecode/lsp_server/services.py @@ -5,7 +5,7 @@ from finecode import domain, user_messages from finecode.config import read_configs from finecode.lsp_server import global_state, schemas -from finecode.runner import manager as runner_manager +from finecode.runner import runner_manager class ActionNotFound(Exception): ... diff --git a/src/finecode/main.py b/src/finecode/main.py deleted file mode 100644 index 6752726..0000000 --- a/src/finecode/main.py +++ /dev/null @@ -1,48 +0,0 @@ -from __future__ import annotations - -from finecode import communication_utils # pygls_server_utils -from finecode import logger_utils -from finecode.lsp_server.lsp_server import create_lsp_server - -# async def start( -# comm_type: communication_utils.CommunicationType, -# host: str | None = None, -# port: int | None = None, -# trace: bool = False, -# ) -> None: -# log_dir_path = Path(app_dirs.get_app_dirs().user_log_dir) -# logger.remove() -# # disable logging raw messages -# # TODO: make configurable -# logger.configure(activation=[("pygls.protocol.json_rpc", False)]) - -# logs.save_logs_to_file( -# file_path=log_dir_path / "execution.log", -# log_level="TRACE" if trace else "INFO", -# stdout=False, -# ) - -# server = create_lsp_server() -# if comm_type == communication_utils.CommunicationType.TCP: -# if host is None or port is None: -# raise ValueError("TCP server requires host and port to be provided.") - -# await pygls_server_utils.start_tcp_async(server, host, port) -# elif comm_type == communication_utils.CommunicationType.WS: -# if host is None or port is None: -# raise ValueError("WS server requires host and port to be provided.") -# raise NotImplementedError() # async version of start_ws is needed -# else: -# # await pygls_utils.start_io_async(server) -# server.start_io() - - -def start_sync( - comm_type: communication_utils.CommunicationType, - host: str | None = None, - port: int | None = None, - trace: bool = False, -) -> None: - logger_utils.init_logger(trace=trace) - server = create_lsp_server() - server.start_io() diff --git a/src/finecode/pygls_client_utils.py b/src/finecode/pygls_client_utils.py deleted file mode 100644 index 7e7a6ce..0000000 --- a/src/finecode/pygls_client_utils.py +++ /dev/null @@ -1,52 +0,0 @@ -import os -import shlex -import subprocess -import sys -from pathlib import Path -from typing import Type - -from pygls.client import JsonRPCClient - -# async def create_lsp_client_tcp(host: str, port: int) -> JsonRPCClient: -# ls = JsonRPCClient() -# await ls.start_tcp(host, port) -# return ls - - -async def create_lsp_client_io( - client_cls: Type[JsonRPCClient], server_cmd: str, working_dir_path: Path -) -> JsonRPCClient: - ls = client_cls() - splitted_cmd = shlex.split(server_cmd) - executable, *args = splitted_cmd - - old_working_dir = os.getcwd() - os.chdir(working_dir_path) - - # temporary remove VIRTUAL_ENV env variable to avoid starting in wrong venv - old_virtual_env_var = os.environ.pop("VIRTUAL_ENV", None) - - creationflags = 0 - # start_new_session = True .. process has parent id of real parent, but is not - # ended if parent was ended - start_new_session = True - if sys.platform == "win32": - # use creationflags because `start_new_session` doesn't work on Windows - # subprocess.CREATE_NO_WINDOW .. no console window on Windows. TODO: test - creationflags = subprocess.DETACHED_PROCESS | subprocess.CREATE_NO_WINDOW - start_new_session = False - - await ls.start_io( - executable, - *args, - start_new_session=start_new_session, - creationflags=creationflags, - ) - if old_virtual_env_var is not None: - os.environ["VIRTUAL_ENV"] = old_virtual_env_var - - os.chdir(old_working_dir) # restore original working directory - return ls - - -__all__ = ["JsonRPCClient", "create_lsp_client_io"] # "create_lsp_client_tcp", diff --git a/src/finecode/pygls_server_utils.py b/src/finecode/pygls_server_utils.py deleted file mode 100644 index 3bfca8d..0000000 --- a/src/finecode/pygls_server_utils.py +++ /dev/null @@ -1,103 +0,0 @@ -import logging -import sys -from threading import Event -from typing import Any, BinaryIO, Optional - -from loguru import logger -from pygls.io_ import StdinAsyncReader, StdoutWriter, run_async -from pygls.lsp.server import LanguageServer - -std_logger = logging.getLogger(__name__) - - -# async def start_tcp_async(server: LanguageServer, host: str, port: int) -> None: -# """Starts TCP server.""" -# logger.info(f"Starting TCP server on {host}:{port}") - -# server._stop_event = stop_event = Event() - -# async def lsp_connection( -# reader: asyncio.StreamReader, writer: asyncio.StreamWriter -# ): -# logger.debug("Connected to client") -# self.protocol.set_writer(writer) # type: ignore -# await run_async( -# stop_event=stop_event, -# reader=reader, -# protocol=server.protocol, -# logger=std_logger, -# error_handler=server.report_server_error, -# ) -# logger.debug("Main loop finished") -# server.shutdown() - -# async def tcp_server(h: str, p: int): -# server._server = await asyncio.start_server(lsp_connection, h, p) - -# addrs = ", ".join(str(sock.getsockname()) for sock in server._server.sockets) -# logger.info(f"Serving on {addrs}") - -# async with server._server: -# await server._server.serve_forever() - -# try: -# await tcp_server(host, port) -# except asyncio.CancelledError: -# logger.debug("Server was cancelled") - - -async def start_io_async( - server: LanguageServer, - stdin: Optional[BinaryIO] = None, - stdout: Optional[BinaryIO] = None, -): - """Starts an asynchronous IO server.""" - logger.info("Starting async IO server") - - server._stop_event = Event() - reader = StdinAsyncReader(stdin or sys.stdin.buffer, server.thread_pool) - writer = StdoutWriter(stdout or sys.stdout.buffer) - server.protocol.set_writer(writer) - - try: - await run_async( - stop_event=server._stop_event, - reader=reader, - protocol=server.protocol, - logger=std_logger, - error_handler=server.report_server_error, - ) - except BrokenPipeError: - logger.error("Connection to the client is lost! Shutting down the server.") - except (KeyboardInterrupt, SystemExit): - pass - finally: - server.shutdown() - - -def deserialize_pygls_object(pygls_object) -> dict[str, Any] | list[Any]: - deserialized: dict[str, Any] | list[Any] - if "_0" in pygls_object._fields: - # list - deserialized = [] - for index in range(len(pygls_object)): - item = getattr(pygls_object, f"_{index}") - if hasattr(item, "__module__") and item.__module__ == "pygls.protocol": - deserialized_value = deserialize_pygls_object(item) - else: - deserialized_value = item - deserialized.append(deserialized_value) - else: - # dict - deserialized = {} - for field_name in pygls_object._fields: - field_value = getattr(pygls_object, field_name) - if ( - hasattr(field_value, "__module__") - and field_value.__module__ == "pygls.protocol" - ): - deserialized_value = deserialize_pygls_object(field_value) - else: - deserialized_value = field_value - deserialized[field_name] = deserialized_value - return deserialized diff --git a/src/finecode/runner/_internal_client_api.py b/src/finecode/runner/_internal_client_api.py new file mode 100644 index 0000000..05ca77c --- /dev/null +++ b/src/finecode/runner/_internal_client_api.py @@ -0,0 +1,72 @@ +""" +Client API used only internally in runner manager or other modules of this package. They +are not intended to be used in higher layers. +""" + +from loguru import logger + +from finecode.runner import _internal_client_types +from finecode.runner.jsonrpc_client import client as jsonrpc_client + + +async def initialize( + client: jsonrpc_client.JsonRpcClient, + client_process_id: int, + client_name: str, + client_version: str, +) -> None: + logger.debug(f"Send initialize to server {client.readable_id}") + await client.send_request( + method=_internal_client_types.INITIALIZE, + params=_internal_client_types.InitializeParams( + process_id=client_process_id, + capabilities=_internal_client_types.ClientCapabilities(), + client_info=_internal_client_types.ClientInfo( + name=client_name, version=client_version + ), + trace=_internal_client_types.TraceValue.Verbose, + ), + timeout=20, + ) + + +async def notify_initialized(client: jsonrpc_client.JsonRpcClient) -> None: + logger.debug(f"Notify initialized {client.readable_id}") + client.notify( + method=_internal_client_types.INITIALIZED, + params=_internal_client_types.InitializedParams(), + ) + + +async def cancel_request( + client: jsonrpc_client.JsonRpcClient, request_id: int | str +) -> None: + logger.debug(f"Cancel request {request_id} | {client.readable_id}") + client.notify( + method=_internal_client_types.CANCEL_REQUEST, + params=_internal_client_types.CancelParams(id=request_id), + ) + + +async def shutdown( + client: jsonrpc_client.JsonRpcClient, +) -> None: + logger.debug(f"Send shutdown to server {client.readable_id}") + await client.send_request(method=_internal_client_types.SHUTDOWN) + + +def shutdown_sync( + client: jsonrpc_client.JsonRpcClient, +) -> None: + logger.debug(f"Send shutdown to server {client.readable_id}") + client.send_request_sync(method=_internal_client_types.SHUTDOWN) + + +async def exit(client: jsonrpc_client.JsonRpcClient) -> None: + logger.debug(f"Send exit to server {client.readable_id}") + client.notify(method=_internal_client_types.EXIT) + + +def exit_sync(client: jsonrpc_client.JsonRpcClient) -> None: + logger.debug(f"Send exit to server {client.readable_id}") + client.notify(method=_internal_client_types.EXIT) diff --git a/src/finecode/runner/_internal_client_types.py b/src/finecode/runner/_internal_client_types.py new file mode 100644 index 0000000..8013f55 --- /dev/null +++ b/src/finecode/runner/_internal_client_types.py @@ -0,0 +1,1503 @@ +""" +Types for ER client. + +LSP were reused where it was meaningful. +""" + +from __future__ import annotations + +import dataclasses +import collections.abc +import enum +import functools +import typing + +EXIT = "exit" +INITIALIZE = "initialize" +INITIALIZED = "initialized" +SHUTDOWN = "shutdown" +CANCEL_REQUEST = "$/cancelRequest" +PROGRESS = "$/progress" +TEXT_DOCUMENT_DID_CLOSE = "textDocument/didClose" +TEXT_DOCUMENT_DID_OPEN = "textDocument/didOpen" +WORKSPACE_EXECUTE_COMMAND = "workspace/executeCommand" +WORKSPACE_APPLY_EDIT = "workspace/applyEdit" + +DOCUMENT_GET = "documents/get" +PROJECT_RAW_CONFIG_GET = "projects/getRawConfig" + + +@dataclasses.dataclass +class BaseRequest: + id: int | str + """The request id.""" + method: str + """The method name.""" + jsonrpc: str + + +@dataclasses.dataclass +class BaseResponse: + id: int | str + """The request id.""" + jsonrpc: str + + +@dataclasses.dataclass +class BaseNotification: + method: str + """The method name.""" + + jsonrpc: str + + +@dataclasses.dataclass +class BaseResult: ... + + +@dataclasses.dataclass +class InitializeParams: + capabilities: ClientCapabilities + """The capabilities provided by the client (editor or tool)""" + + process_id: int | None = None + """The process Id of the parent process that started + the server. + + Is `null` if the process has not been started by another process. + If the parent process is not alive then the server should exit.""" + + client_info: ClientInfo | None = None + """Information about the client + + @since 3.15.0""" + # Since: 3.15.0 + + locale: str | None = None + """The locale the client is currently showing the user interface + in. This must not necessarily be the locale of the operating + system. + + Uses IETF language tags as the value's syntax + (See https://en.wikipedia.org/wiki/IETF_language_tag) + + @since 3.16.0""" + # Since: 3.16.0 + + root_path: str | None = None + """The rootPath of the workspace. Is null + if no folder is open. + + @deprecated in favour of rootUri.""" + + root_uri: str | None = None + """The rootUri of the workspace. Is null if no + folder is open. If both `rootPath` and `rootUri` are set + `rootUri` wins. + + @deprecated in favour of workspaceFolders.""" + + initialization_options: LSPAny | None = None + """User provided initialization options.""" + + trace: TraceValue | None = None + """The initial trace setting. If omitted trace is disabled ('off').""" + + work_done_token: ProgressToken | None = None + """An optional token that a server can use to report work done progress.""" + + workspace_folders: collections.abc.Sequence[WorkspaceFolder] | None = None + """The workspace folders configured in the client when the server starts. + + This property is only available if the client supports workspace folders. + It can be `null` if the client supports workspace folders but none are + configured. + + @since 3.6.0""" + # Since: 3.6.0 + + +@dataclasses.dataclass +class InitializeRequest(BaseRequest): + params: InitializeParams + method = "initialize" + + +@dataclasses.dataclass +class InitializeResult(BaseResult): + """The result returned from an initialize request.""" + + capabilities: ServerCapabilities + """The capabilities the language server provides.""" + + server_info: ServerInfo | None = None + """Information about the server. + + @since 3.15.0""" + # Since: 3.15.0 + + +@dataclasses.dataclass +class InitializeResponse(BaseResponse): + result: InitializeResult + + +@dataclasses.dataclass +class InitializeError: + """The data type of the ResponseError if the + initialize request fails.""" + + retry: bool + """Indicates whether the client execute the following retry logic: + (1) show the message provided by the ResponseError to the user + (2) user selects retry or cancel + (3) if user selected retry the initialize method is sent again.""" + + +@dataclasses.dataclass +class InitializedParams: + pass + + +@dataclasses.dataclass +class ClientCapabilities: + """Defines the capabilities provided by the client.""" + + # workspace: WorkspaceClientCapabilities | None = None + """Workspace specific client capabilities.""" + + # text_document: TextDocumentClientCapabilities | None = None + """Text document specific client capabilities.""" + + # notebook_document: NotebookDocumentClientCapabilities | None = None + """Capabilities specific to the notebook document support. + + @since 3.17.0""" + # Since: 3.17.0 + + # window: WindowClientCapabilities | None = None + """Window specific client capabilities.""" + + # general: GeneralClientCapabilities | None = None + """General client capabilities. + + @since 3.16.0""" + # Since: 3.16.0 + + experimental: LSPAny | None = None + """Experimental client capabilities.""" + + +@dataclasses.dataclass +class ClientInfo: + """Information about the client + + @since 3.15.0 + @since 3.18.0 ClientInfo type name added.""" + + # Since: + # 3.15.0 + # 3.18.0 ClientInfo type name added. + + name: str + """The name of the client as defined by the client.""" + + version: str | None = None + """The client's version as defined by the client.""" + + +LSPAny = typing.Any | None +"""The LSP any type. +Please note that strictly speaking a property with the value `undefined` +can't be converted into JSON preserving the property name. However for +convenience it is allowed and assumed that all these properties are +optional as well. +@since 3.17.0""" +# Since: 3.17.0 + + +@enum.unique +class TraceValue(str, enum.Enum): + Off = "off" + """Turn tracing off.""" + Messages = "messages" + """Trace messages only.""" + Verbose = "verbose" + """Verbose message tracing.""" + + +ProgressToken = int | str + + +@dataclasses.dataclass +class WorkspaceFolder: + """A workspace folder inside a client.""" + + uri: str + """The associated URI for this workspace folder.""" + + name: str + """The name of the workspace folder. Used to refer to this + workspace folder in the user interface.""" + + +@enum.unique +class PositionEncodingKind(str, enum.Enum): + """A set of predefined position encoding kinds. + + @since 3.17.0""" + + # Since: 3.17.0 + Utf8 = "utf-8" + """Character offsets count UTF-8 code units (e.g. bytes).""" + Utf16 = "utf-16" + """Character offsets count UTF-16 code units. + + This is the default and must always be supported + by servers""" + Utf32 = "utf-32" + """Character offsets count UTF-32 code units. + + Implementation note: these are the same as Unicode codepoints, + so this `PositionEncodingKind` may also be used for an + encoding-agnostic representation of character offsets.""" + + +@dataclasses.dataclass +class SaveOptions: + """Save options.""" + + include_text: bool | None = None + """The client is supposed to include the content on save.""" + + +@dataclasses.dataclass +class TextDocumentSyncOptions: + open_close: bool | None = None + """Open and close notifications are sent to the server. If omitted open close notification should not + be sent.""" + + change: TextDocumentSyncKind | None = None + """Change notifications are sent to the server. See TextDocumentSyncKind.None, TextDocumentSyncKind.Full + and TextDocumentSyncKind.Incremental. If omitted it defaults to TextDocumentSyncKind.None.""" + + will_save: bool | None = None + """If present will save notifications are sent to the server. If omitted the notification should not be + sent.""" + + will_save_wait_until: bool | None = None + """If present will save wait until requests are sent to the server. If omitted the request should not be + sent.""" + + save: bool | SaveOptions | None = None + """If present save notifications are sent to the server. If omitted the notification should not be + sent.""" + + +@enum.unique +class TextDocumentSyncKind(int, enum.Enum): + """Defines how the host (editor) should sync + document changes to the language server.""" + + None_ = 0 + """Documents should not be synced at all.""" + Full = 1 + """Documents are synced by always sending the full content + of the document.""" + Incremental = 2 + """Documents are synced by sending the full content on open. + After that only incremental updates to the document are + send.""" + + +@dataclasses.dataclass +class ExecuteCommandOptions: + """The server capabilities of a {@link ExecuteCommandRequest}.""" + + commands: collections.abc.Sequence[str] + """The commands to be executed on the server""" + + work_done_progress: bool | None = None + + +@dataclasses.dataclass +class WorkspaceFoldersServerCapabilities: + supported: bool | None = None + """The server has support for workspace folders""" + + change_notifications: str | bool | None = None + """Whether the server wants to receive workspace folder + change notifications. + + If a string is provided the string is treated as an ID + under which the notification is registered on the client + side. The ID can be used to unregister for these events + using the `client/unregisterCapability` request.""" + + +@enum.unique +class FileOperationPatternKind(str, enum.Enum): + """A pattern kind describing if a glob pattern matches a file a folder or + both. + + @since 3.16.0""" + + # Since: 3.16.0 + File = "file" + """The pattern matches a file only.""" + Folder = "folder" + """The pattern matches a folder only.""" + + +@dataclasses.dataclass +class FileOperationPatternOptions: + """Matching options for the file operation pattern. + + @since 3.16.0""" + + # Since: 3.16.0 + + ignore_case: bool | None = None + """The pattern should be matched ignoring casing.""" + + +@dataclasses.dataclass +class FileOperationPattern: + """A pattern to describe in which file operation requests or notifications + the server is interested in receiving. + + @since 3.16.0""" + + # Since: 3.16.0 + + glob: str + """The glob pattern to match. Glob patterns can have the following syntax: + - `*` to match one or more characters in a path segment + - `?` to match on one character in a path segment + - `**` to match any number of path segments, including none + - `{}` to group sub patterns into an OR expression. (e.g. `**/*.{ts,js}` matches all TypeScript and JavaScript files) + - `[]` to declare a range of characters to match in a path segment (e.g., `example.[0-9]` to match on `example.0`, `example.1`, …) + - `[!...]` to negate a range of characters to match in a path segment (e.g., `example.[!0-9]` to match on `example.a`, `example.b`, but not `example.0`)""" + + matches: FileOperationPatternKind | None = None + """Whether to match files or folders with this pattern. + + Matches both if undefined.""" + + options: FileOperationPatternOptions | None = None + """Additional options used during matching.""" + + +@dataclasses.dataclass +class FileOperationFilter: + """A filter to describe in which file operation requests or notifications + the server is interested in receiving. + + @since 3.16.0""" + + # Since: 3.16.0 + + pattern: FileOperationPattern + """The actual file operation pattern.""" + + scheme: str | None = None + """A Uri scheme like `file` or `untitled`.""" + + +@dataclasses.dataclass +class FileOperationRegistrationOptions: + """The options to register for file operations. + + @since 3.16.0""" + + # Since: 3.16.0 + + filters: collections.abc.Sequence[FileOperationFilter] + """The actual filters.""" + + +@dataclasses.dataclass +class FileOperationOptions: + """Options for notifications/requests for user operations on files. + + @since 3.16.0""" + + # Since: 3.16.0 + + did_create: FileOperationRegistrationOptions | None = None + """The server is interested in receiving didCreateFiles notifications.""" + + will_create: FileOperationRegistrationOptions | None = None + """The server is interested in receiving willCreateFiles requests.""" + + did_rename: FileOperationRegistrationOptions | None = None + """The server is interested in receiving didRenameFiles notifications.""" + + will_rename: FileOperationRegistrationOptions | None = None + """The server is interested in receiving willRenameFiles requests.""" + + did_delete: FileOperationRegistrationOptions | None = None + """The server is interested in receiving didDeleteFiles file notifications.""" + + will_delete: FileOperationRegistrationOptions | None = None + """The server is interested in receiving willDeleteFiles file requests.""" + + +@dataclasses.dataclass +class TextDocumentContentOptions: + """Text document content provider options. + + @since 3.18.0 + @proposed""" + + # Since: 3.18.0 + # Proposed + + schemes: collections.abc.Sequence[str] + """The schemes for which the server provides content.""" + + +@dataclasses.dataclass +class TextDocumentContentRegistrationOptions: + """Text document content provider registration options. + + @since 3.18.0 + @proposed""" + + # Since: 3.18.0 + # Proposed + + schemes: collections.abc.Sequence[str] + """The schemes for which the server provides content.""" + + id: str | None = None + """The id used to register the request. The id can be used to deregister + the request again. See also Registration#id.""" + + +@dataclasses.dataclass +class WorkspaceOptions: + """Defines workspace specific capabilities of the server. + + @since 3.18.0""" + + # Since: 3.18.0 + + workspace_folders: WorkspaceFoldersServerCapabilities | None = None + """The server supports workspace folder. + + @since 3.6.0""" + # Since: 3.6.0 + + file_operations: FileOperationOptions | None = None + """The server is interested in notifications/requests for operations on files. + + @since 3.16.0""" + # Since: 3.16.0 + + text_document_content: ( + TextDocumentContentOptions | TextDocumentContentRegistrationOptions | None + ) = None + """The server supports the `workspace/textDocumentContent` request. + + @since 3.18.0 + @proposed""" + # Since: 3.18.0 + # Proposed + + +@dataclasses.dataclass +class ServerCapabilities: + """Defines the capabilities provided by a language + server.""" + + position_encoding: PositionEncodingKind | str | None = None + """The position encoding the server picked from the encodings offered + by the client via the client capability `general.positionEncodings`. + + If the client didn't provide any position encodings the only valid + value that a server can return is 'utf-16'. + + If omitted it defaults to 'utf-16'. + + @since 3.17.0""" + # Since: 3.17.0 + + text_document_sync: TextDocumentSyncOptions | TextDocumentSyncKind | None = None + """Defines how text documents are synced. Is either a detailed structure + defining each notification or for backwards compatibility the + TextDocumentSyncKind number.""" + + # notebook_document_sync: Optional[ + # Union[NotebookDocumentSyncOptions, NotebookDocumentSyncRegistrationOptions] + # ] = attrs.field(default=None) + """Defines how notebook documents are synced. + + @since 3.17.0""" + # Since: 3.17.0 + + # completion_provider: Optional[CompletionOptions] = attrs.field(default=None) + """The server provides completion support.""" + + # hover_provider: Optional[Union[bool, HoverOptions]] = attrs.field(default=None) + """The server provides hover support.""" + + # signature_help_provider: Optional[SignatureHelpOptions] = attrs.field(default=None) + """The server provides signature help support.""" + + # declaration_provider: Optional[ + # Union[bool, DeclarationOptions, DeclarationRegistrationOptions] + # ] = attrs.field(default=None) + """The server provides Goto Declaration support.""" + + # definition_provider: Optional[Union[bool, DefinitionOptions]] = attrs.field( + # default=None + # ) + """The server provides goto definition support.""" + + # type_definition_provider: Optional[ + # Union[bool, TypeDefinitionOptions, TypeDefinitionRegistrationOptions] + # ] = attrs.field(default=None) + """The server provides Goto Type Definition support.""" + + # implementation_provider: Optional[ + # Union[bool, ImplementationOptions, ImplementationRegistrationOptions] + # ] = attrs.field(default=None) + """The server provides Goto Implementation support.""" + + # references_provider: Optional[Union[bool, ReferenceOptions]] = attrs.field( + # default=None + # ) + """The server provides find references support.""" + + # document_highlight_provider: Optional[Union[bool, DocumentHighlightOptions]] = ( + # attrs.field(default=None) + # ) + """The server provides document highlight support.""" + + # document_symbol_provider: Optional[Union[bool, DocumentSymbolOptions]] = ( + # attrs.field(default=None) + # ) + """The server provides document symbol support.""" + + # code_action_provider: Optional[Union[bool, CodeActionOptions]] = attrs.field( + # default=None + # ) + """The server provides code actions. CodeActionOptions may only be + specified if the client states that it supports + `codeActionLiteralSupport` in its initial `initialize` request.""" + + # code_lens_provider: Optional[CodeLensOptions] = attrs.field(default=None) + """The server provides code lens.""" + + # document_link_provider: Optional[DocumentLinkOptions] = attrs.field(default=None) + """The server provides document link support.""" + + # color_provider: Optional[ + # Union[bool, DocumentColorOptions, DocumentColorRegistrationOptions] + # ] = attrs.field(default=None) + """The server provides color provider support.""" + + # workspace_symbol_provider: Optional[Union[bool, WorkspaceSymbolOptions]] = ( + # attrs.field(default=None) + # ) + """The server provides workspace symbol support.""" + + # document_formatting_provider: Optional[Union[bool, DocumentFormattingOptions]] = ( + # attrs.field(default=None) + # ) + """The server provides document formatting.""" + + # document_range_formatting_provider: Optional[ + # Union[bool, DocumentRangeFormattingOptions] + # ] = attrs.field(default=None) + """The server provides document range formatting.""" + + # document_on_type_formatting_provider: Optional[DocumentOnTypeFormattingOptions] = ( + # attrs.field(default=None) + # ) + """The server provides document formatting on typing.""" + + # rename_provider: Optional[Union[bool, RenameOptions]] = attrs.field(default=None) + """The server provides rename support. RenameOptions may only be + specified if the client states that it supports + `prepareSupport` in its initial `initialize` request.""" + + # folding_range_provider: Optional[ + # Union[bool, FoldingRangeOptions, FoldingRangeRegistrationOptions] + # ] = attrs.field(default=None) + """The server provides folding provider support.""" + + # selection_range_provider: Optional[ + # Union[bool, SelectionRangeOptions, SelectionRangeRegistrationOptions] + # ] = attrs.field(default=None) + """The server provides selection range support.""" + + execute_command_provider: ExecuteCommandOptions | None = None + """The server provides execute command support.""" + + # call_hierarchy_provider: Optional[ + # Union[bool, CallHierarchyOptions, CallHierarchyRegistrationOptions] + # ] = attrs.field(default=None) + """The server provides call hierarchy support. + + @since 3.16.0""" + # Since: 3.16.0 + + # linked_editing_range_provider: Optional[ + # Union[bool, LinkedEditingRangeOptions, LinkedEditingRangeRegistrationOptions] + # ] = attrs.field(default=None) + """The server provides linked editing range support. + + @since 3.16.0""" + # Since: 3.16.0 + + # semantic_tokens_provider: Optional[ + # Union[SemanticTokensOptions, SemanticTokensRegistrationOptions] + # ] = attrs.field(default=None) + """The server provides semantic tokens support. + + @since 3.16.0""" + # Since: 3.16.0 + + # moniker_provider: Optional[ + # Union[bool, MonikerOptions, MonikerRegistrationOptions] + # ] = attrs.field(default=None) + """The server provides moniker support. + + @since 3.16.0""" + # Since: 3.16.0 + + # type_hierarchy_provider: Optional[ + # Union[bool, TypeHierarchyOptions, TypeHierarchyRegistrationOptions] + # ] = attrs.field(default=None) + """The server provides type hierarchy support. + + @since 3.17.0""" + # Since: 3.17.0 + + # inline_value_provider: Optional[ + # Union[bool, InlineValueOptions, InlineValueRegistrationOptions] + # ] = attrs.field(default=None) + """The server provides inline values. + + @since 3.17.0""" + # Since: 3.17.0 + + # inlay_hint_provider: Optional[ + # Union[bool, InlayHintOptions, InlayHintRegistrationOptions] + # ] = attrs.field(default=None) + """The server provides inlay hints. + + @since 3.17.0""" + # Since: 3.17.0 + + # diagnostic_provider: Optional[ + # Union[DiagnosticOptions, DiagnosticRegistrationOptions] + # ] = attrs.field(default=None) + """The server has support for pull model diagnostics. + + @since 3.17.0""" + # Since: 3.17.0 + + # inline_completion_provider: Optional[Union[bool, InlineCompletionOptions]] = ( + # attrs.field(default=None) + # ) + """Inline completion options used during static registration. + + @since 3.18.0 + @proposed""" + # Since: 3.18.0 + # Proposed + + workspace: WorkspaceOptions | None = None + """Workspace specific server capabilities.""" + + experimental: LSPAny | None = None + """Experimental server capabilities.""" + + +@dataclasses.dataclass +class ServerInfo: + """Information about the server + + @since 3.15.0 + @since 3.18.0 ServerInfo type name added.""" + + # Since: + # 3.15.0 + # 3.18.0 ServerInfo type name added. + + name: str + """The name of the server as defined by the server.""" + + version: str | None = None + """The server's version as defined by the server.""" + + +@dataclasses.dataclass +class ExecuteCommandParams: + """The parameters of a {@link ExecuteCommandRequest}.""" + + command: str + """The identifier of the actual command handler.""" + + arguments: collections.abc.Sequence[LSPAny] | None = None + """Arguments that the command should be invoked with.""" + + work_done_token: ProgressToken | None = None + """An optional token that a server can use to report work done progress.""" + + +@dataclasses.dataclass +class ExecuteCommandRequest(BaseRequest): + params: ExecuteCommandParams + method = "workspace/executeCommand" + + +@dataclasses.dataclass +class ExecuteCommandResponse(BaseResponse): + result: LSPAny | None = None + + +@dataclasses.dataclass +class DidOpenTextDocumentParams: + """The parameters sent in an open text document notification""" + + text_document: TextDocumentItem + """The document that was opened.""" + + +@dataclasses.dataclass +class TextDocumentItem: + """An item to transfer a text document from the client to the + server.""" + + uri: str + """The text document's uri.""" + + language_id: LanguageKind | str + """The text document's language identifier.""" + + version: int + """The version number of this document (it will increase after each + change, including undo/redo).""" + + text: str + """The content of the opened text document.""" + + +class LanguageKind(str, enum.Enum): + """Predefined Language kinds + @since 3.18.0""" + + # Since: 3.18.0 + Abap = "abap" + WindowsBat = "bat" + BibTeX = "bibtex" + Clojure = "clojure" + Coffeescript = "coffeescript" + C = "c" + Cpp = "cpp" + CSharp = "csharp" + Css = "css" + D = "d" + """@since 3.18.0 + @proposed""" + # Since: 3.18.0 + # Proposed + Delphi = "pascal" + """@since 3.18.0 + @proposed""" + # Since: 3.18.0 + # Proposed + Diff = "diff" + Dart = "dart" + Dockerfile = "dockerfile" + Elixir = "elixir" + Erlang = "erlang" + FSharp = "fsharp" + GitCommit = "git-commit" + GitRebase = "rebase" + Go = "go" + Groovy = "groovy" + Handlebars = "handlebars" + Haskell = "haskell" + Html = "html" + Ini = "ini" + Java = "java" + JavaScript = "javascript" + JavaScriptReact = "javascriptreact" + Json = "json" + LaTeX = "latex" + Less = "less" + Lua = "lua" + Makefile = "makefile" + Markdown = "markdown" + ObjectiveC = "objective-c" + ObjectiveCpp = "objective-cpp" + Pascal = "pascal" + """@since 3.18.0 + @proposed""" + # Since: 3.18.0 + # Proposed + Perl = "perl" + Perl6 = "perl6" + Php = "php" + Powershell = "powershell" + Pug = "jade" + Python = "python" + R = "r" + Razor = "razor" + Ruby = "ruby" + Rust = "rust" + Scss = "scss" + Sass = "sass" + Scala = "scala" + ShaderLab = "shaderlab" + ShellScript = "shellscript" + Sql = "sql" + Swift = "swift" + TypeScript = "typescript" + TypeScriptReact = "typescriptreact" + TeX = "tex" + VisualBasic = "vb" + Xml = "xml" + Xsl = "xsl" + Yaml = "yaml" + + +@dataclasses.dataclass +class DidCloseTextDocumentParams: + """The parameters sent in a close text document notification""" + + text_document: TextDocumentIdentifier + """The document that was closed.""" + + +@dataclasses.dataclass +class TextDocumentIdentifier: + """A literal to identify a text document in the client.""" + + uri: str + """The text document's uri.""" + + +@dataclasses.dataclass +class ProgressParams: + token: ProgressToken + """The progress token provided by the client or server.""" + + value: LSPAny + """The progress data.""" + + +@dataclasses.dataclass +class ProgressNotification(BaseNotification): + params: ProgressParams + method = "$/progress" + + +@dataclasses.dataclass +class ApplyWorkspaceEditParams: + """The parameters passed via an apply workspace edit request.""" + + edit: WorkspaceEdit + """The edits to apply.""" + + label: str | None = None + """An optional label of the workspace edit. This label is + presented in the user interface for example on an undo + stack to undo the workspace edit.""" + + metadata: WorkspaceEditMetadata | None = None + """Additional data about the edit. + + @since 3.18.0 + @proposed""" + # Since: 3.18.0 + # Proposed + + +@dataclasses.dataclass +class ApplyWorkspaceEditRequest(BaseRequest): + params: ApplyWorkspaceEditParams + method = "workspace/applyEdit" + + +@dataclasses.dataclass +class ApplyWorkspaceEditResult(BaseResult): + """The result returned from the apply workspace edit request. + + @since 3.17 renamed from ApplyWorkspaceEditResponse""" + + # Since: 3.17 renamed from ApplyWorkspaceEditResponse + + applied: bool + """Indicates whether the edit was applied or not.""" + + failure_reason: str | None = None + """An optional textual description for why the edit was not applied. + This may be used by the server for diagnostic logging or to provide + a suitable error for a request that triggered the edit.""" + + failed_change: int | None = None + """Depending on the client's failure handling strategy `failedChange` might + contain the index of the change that failed. This property is only available + if the client signals a `failureHandlingStrategy` in its client capabilities.""" + + +@dataclasses.dataclass +class ApplyWorkspaceEditResponse(BaseResponse): + result: ApplyWorkspaceEditResult + + +@dataclasses.dataclass +class WorkspaceEdit: + """A workspace edit represents changes to many resources managed in the workspace. The edit + should either provide `changes` or `documentChanges`. If documentChanges are present + they are preferred over `changes` if the client can handle versioned document edits. + + Since version 3.13.0 a workspace edit can contain resource operations as well. If resource + operations are present clients need to execute the operations in the order in which they + are provided. So a workspace edit for example can consist of the following two changes: + (1) a create file a.txt and (2) a text document edit which insert text into file a.txt. + + An invalid sequence (e.g. (1) delete file a.txt and (2) insert text into file a.txt) will + cause failure of the operation. How the client recovers from the failure is described by + the client capability: `workspace.workspaceEdit.failureHandling`""" + + changes: collections.abc.Mapping[str, collections.abc.Sequence[TextEdit]] | None = ( + None + ) + """Holds changes to existing resources.""" + + document_changes: ( + collections.abc.Sequence[ + TextDocumentEdit | CreateFile | RenameFile | DeleteFile + ] + | None + ) = None + """Depending on the client capability `workspace.workspaceEdit.resourceOperations` document changes + are either an array of `TextDocumentEdit`s to express changes to n different text documents + where each text document edit addresses a specific version of a text document. Or it can contain + above `TextDocumentEdit`s mixed with create, rename and delete file / folder operations. + + Whether a client supports versioned document edits is expressed via + `workspace.workspaceEdit.documentChanges` client capability. + + If a client neither supports `documentChanges` nor `workspace.workspaceEdit.resourceOperations` then + only plain `TextEdit`s using the `changes` property are supported.""" + + change_annotations: ( + collections.abc.Mapping[ChangeAnnotationIdentifier, ChangeAnnotation] | None + ) = None + """A map of change annotations that can be referenced in `AnnotatedTextEdit`s or create, rename and + delete file / folder operations. + + Whether clients honor this property depends on the client capability `workspace.changeAnnotationSupport`. + + @since 3.16.0""" + # Since: 3.16.0 + + +@dataclasses.dataclass +class CreateFile: + """Create file operation.""" + + uri: str + """The resource to create.""" + + kind: typing.Literal["create"] = "create" + """A create""" + + options: CreateFileOptions | None = None + """Additional options""" + + annotation_id: ChangeAnnotationIdentifier | None = None + """An optional annotation identifier describing the operation. + + @since 3.16.0""" + # Since: 3.16.0 + + +@dataclasses.dataclass +class CreateFileOptions: + """Options to create a file.""" + + overwrite: bool | None = None + """Overwrite existing file. Overwrite wins over `ignoreIfExists`""" + + ignore_if_exists: bool | None = None + """Ignore if exists.""" + + +ChangeAnnotationIdentifier = str + + +@dataclasses.dataclass +class RenameFile: + """Rename file operation""" + + old_uri: str + """The old (existing) location.""" + + new_uri: str + """The new location.""" + + kind: typing.Literal["rename"] = "rename" + """A rename""" + + options: RenameFileOptions | None = None + """Rename options.""" + + annotation_id: ChangeAnnotationIdentifier | None = None + """An optional annotation identifier describing the operation. + + @since 3.16.0""" + # Since: 3.16.0 + + +@dataclasses.dataclass +class RenameFileOptions: + """Rename file options""" + + overwrite: bool | None = None + """Overwrite target if existing. Overwrite wins over `ignoreIfExists`""" + + ignore_if_exists: bool | None = None + """Ignores if target exists.""" + + +@dataclasses.dataclass +class DeleteFile: + """Delete file operation""" + + uri: str + """The file to delete.""" + + kind: typing.Literal["delete"] = "delete" + """A delete""" + + options: DeleteFileOptions | None = None + """Delete options.""" + + annotation_id: ChangeAnnotationIdentifier | None = None + """An optional annotation identifier describing the operation. + + @since 3.16.0""" + # Since: 3.16.0 + + +@dataclasses.dataclass +class DeleteFileOptions: + """Delete file options""" + + recursive: bool | None = None + """Delete the content recursively if a folder is denoted.""" + + ignore_if_not_exists: bool | None = None + """Ignore the operation if the file doesn't exist.""" + + +@dataclasses.dataclass +class ChangeAnnotation: + """Additional information that describes document changes. + + @since 3.16.0""" + + # Since: 3.16.0 + + label: str + """A human-readable string describing the actual change. The string + is rendered prominent in the user interface.""" + + needs_confirmation: bool | None = None + """A flag which indicates that user confirmation is needed + before applying the change.""" + + description: str | None = None + """A human-readable string which is rendered less prominent in + the user interface.""" + + +@dataclasses.dataclass +class TextEdit: + """A text edit applicable to a text document.""" + + range: Range + """The range of the text document to be manipulated. To insert + text into a document create a range where start === end.""" + + new_text: str + """The string to be inserted. For delete operations use an + empty string.""" + + +@dataclasses.dataclass +class Range: + """A range in a text document expressed as (zero-based) start and end positions. + + If you want to specify a range that contains a line including the line ending + character(s) then use an end position denoting the start of the next line. + For example: + ```ts + { + start: { line: 5, character: 23 } + end : { line 6, character : 0 } + } + ```""" + + start: Position + """The range's start position.""" + + end: Position + """The range's end position.""" + + @typing.override + def __eq__(self, o: object) -> bool: + if not isinstance(o, Range): + return NotImplemented + return (self.start == o.start) and (self.end == o.end) + + @typing.override + def __repr__(self) -> str: + return f"{self.start!r}-{self.end!r}" + + +@dataclasses.dataclass +@functools.total_ordering +class Position: + """Position in a text document expressed as zero-based line and character + offset. Prior to 3.17 the offsets were always based on a UTF-16 string + representation. So a string of the form `a𐐀b` the character offset of the + character `a` is 0, the character offset of `𐐀` is 1 and the character + offset of b is 3 since `𐐀` is represented using two code units in UTF-16. + Since 3.17 clients and servers can agree on a different string encoding + representation (e.g. UTF-8). The client announces it's supported encoding + via the client capability [`general.positionEncodings`](https://microsoft.github.io/language-server-protocol/specifications/specification-current/#clientCapabilities). + The value is an array of position encodings the client supports, with + decreasing preference (e.g. the encoding at index `0` is the most preferred + one). To stay backwards compatible the only mandatory encoding is UTF-16 + represented via the string `utf-16`. The server can pick one of the + encodings offered by the client and signals that encoding back to the + client via the initialize result's property + [`capabilities.positionEncoding`](https://microsoft.github.io/language-server-protocol/specifications/specification-current/#serverCapabilities). If the string value + `utf-16` is missing from the client's capability `general.positionEncodings` + servers can safely assume that the client supports UTF-16. If the server + omits the position encoding in its initialize result the encoding defaults + to the string value `utf-16`. Implementation considerations: since the + conversion from one encoding into another requires the content of the + file / line the conversion is best done where the file is read which is + usually on the server side. + + Positions are line end character agnostic. So you can not specify a position + that denotes `\r|\n` or `\n|` where `|` represents the character offset. + + @since 3.17.0 - support for negotiated position encoding.""" + + # Since: 3.17.0 - support for negotiated position encoding. + + line: int + """Line position in a document (zero-based).""" + + character: int + """Character offset on a line in a document (zero-based). + + The meaning of this offset is determined by the negotiated + `PositionEncodingKind`.""" + + @typing.override + def __eq__(self, o: object) -> bool: + if not isinstance(o, Position): + return NotImplemented + return (self.line, self.character) == (o.line, o.character) + + def __gt__(self, o: object) -> bool: + if not isinstance(o, Position): + return NotImplemented + return (self.line, self.character) > (o.line, o.character) + + @typing.override + def __repr__(self) -> str: + return f"{self.line}:{self.character}" + + +@dataclasses.dataclass +class WorkspaceEditMetadata: + """Additional data about a workspace edit. + + @since 3.18.0 + @proposed""" + + # Since: 3.18.0 + # Proposed + + is_refactoring: bool | None = None + """Signal to the editor that this edit is a refactoring.""" + + +@dataclasses.dataclass +class OptionalVersionedTextDocumentIdentifier: + """A text document identifier to optionally denote a specific version of a text document.""" + + uri: str + """The text document's uri.""" + + version: int | None = None + """The version number of this document. If a versioned text document identifier + is sent from the server to the client and the file is not open in the editor + (the server has not received an open notification before) the server can send + `null` to indicate that the version is unknown and the content on disk is the + truth (as specified with document content ownership).""" + + +@dataclasses.dataclass +class TextDocumentEdit: + """Describes textual changes on a text document. A TextDocumentEdit describes all changes + on a document version Si and after they are applied move the document to version Si+1. + So the creator of a TextDocumentEdit doesn't need to sort the array of edits or do any + kind of ordering. However the edits must be non overlapping.""" + + text_document: OptionalVersionedTextDocumentIdentifier + """The text document to change.""" + + edits: collections.abc.Sequence[TextEdit | AnnotatedTextEdit | SnippetTextEdit] + """The edits to be applied. + + @since 3.16.0 - support for AnnotatedTextEdit. This is guarded using a + client capability. + + @since 3.18.0 - support for SnippetTextEdit. This is guarded using a + client capability.""" + # Since: + # 3.16.0 - support for AnnotatedTextEdit. This is guarded using a client capability. + # 3.18.0 - support for SnippetTextEdit. This is guarded using a client capability. + + +@dataclasses.dataclass +class AnnotatedTextEdit: + """A special text edit with an additional change annotation. + + @since 3.16.0.""" + + # Since: 3.16.0. + + annotation_id: ChangeAnnotationIdentifier + """The actual identifier of the change annotation""" + + range: Range + """The range of the text document to be manipulated. To insert + text into a document create a range where start === end.""" + + new_text: str + """The string to be inserted. For delete operations use an + empty string.""" + + +@dataclasses.dataclass +class SnippetTextEdit: + """An interactive text edit. + + @since 3.18.0 + @proposed""" + + # Since: 3.18.0 + # Proposed + + range: Range + """The range of the text document to be manipulated.""" + + snippet: StringValue + """The snippet to be inserted.""" + + annotation_id: ChangeAnnotationIdentifier | None = None + """The actual identifier of the snippet edit.""" + + +@dataclasses.dataclass +class StringValue: + """A string value used as a snippet is a template which allows to insert text + and to control the editor cursor when insertion happens. + + A snippet can define tab stops and placeholders with `$1`, `$2` + and `${3:foo}`. `$0` defines the final tab stop, it defaults to + the end of the snippet. Variables are defined with `$name` and + `${name:default value}`. + + @since 3.18.0 + @proposed""" + + # Since: 3.18.0 + # Proposed + + value: str + """The snippet string.""" + + kind: typing.Literal["snippet"] = "snippet" + """The kind of string value.""" + + +@dataclasses.dataclass +class GetDocumentParams: + uri: str + + +@dataclasses.dataclass +class GetDocumentRequest(BaseRequest): + params: GetDocumentParams + method = "documents/get" + + +@dataclasses.dataclass +class GetDocumentResult(BaseResult): + uri: str + version: str + text: str + + +@dataclasses.dataclass +class GetDocumentResponse(BaseResponse): + result: GetDocumentResult + + +@dataclasses.dataclass +class GetProjectRawConfigParams: + project_def_path: str + + +@dataclasses.dataclass +class GetProjectRawConfigRequest(BaseRequest): + params: GetProjectRawConfigParams + method = "projects/getRawConfig" + + +@dataclasses.dataclass +class GetProjectRawConfigResult(BaseResult): + # stringified json + config: str + + +@dataclasses.dataclass +class GetProjectRawConfigResponse(BaseResponse): + result: GetProjectRawConfigResult + + +@dataclasses.dataclass +class InitializedNotification(BaseNotification): + """The initialized notification is sent from the client to the + server after the client is fully initialized and the server + is allowed to send requests from the server to the client.""" + + params: InitializedParams + + method = "initialized" + """The method to be invoked.""" + + +@dataclasses.dataclass +class DidOpenTextDocumentNotification(BaseNotification): + """The document open notification is sent from the client to the server to signal + newly opened text documents. The document's truth is now managed by the client + and the server must not try to read the document's truth using the document's + uri. Open in this sense means it is managed by the client. It doesn't necessarily + mean that its content is presented in an editor. An open notification must not + be sent more than once without a corresponding close notification send before. + This means open and close notification must be balanced and the max open count + is one.""" + + params: DidOpenTextDocumentParams + method = "textDocument/didOpen" + + +@dataclasses.dataclass +class DidCloseTextDocumentNotification(BaseNotification): + """The document close notification is sent from the client to the server when + the document got closed in the client. The document's truth now exists where + the document's uri points to (e.g. if the document's uri is a file uri the + truth now exists on disk). As with the open notification the close notification + is about managing the document's content. Receiving a close notification + doesn't mean that the document was open in an editor before. A close + notification requires a previous open notification to be sent.""" + + params: DidOpenTextDocumentParams + method = "textDocument/didClose" + + +@dataclasses.dataclass +class CancelParams: + id: int | str + """The request id to cancel.""" + + +@dataclasses.dataclass +class CancelNotification(BaseNotification): + params: CancelParams + method = "$/cancelRequest" + + +@dataclasses.dataclass +class ShutdownRequest(BaseRequest): + params: None = None + + +@dataclasses.dataclass +class ShutdownResponse(BaseResponse): ... + + +@dataclasses.dataclass +class ExitNotification(BaseNotification): + params: None = None + + method = "exit" + """The method to be invoked.""" + + +METHOD_TO_TYPES: dict[ + str, + tuple[type[BaseRequest], type | None, type[BaseResponse], type[BaseResult] | None] + | tuple[type[BaseNotification], type | None, None, None], +] = { + INITIALIZE: ( + InitializeRequest, + InitializeParams, + InitializeResponse, + InitializeResult, + ), + INITIALIZED: (InitializedNotification, InitializedParams, None, None), + CANCEL_REQUEST: (CancelNotification, CancelParams, None, None), + SHUTDOWN: (ShutdownRequest, None, ShutdownResponse, None), + PROGRESS: (ProgressNotification, ProgressParams, None, None), + EXIT: (ExitNotification, None, None, None), + WORKSPACE_EXECUTE_COMMAND: ( + ExecuteCommandRequest, + ExecuteCommandParams, + ExecuteCommandResponse, + None, + ), + WORKSPACE_APPLY_EDIT: ( + ApplyWorkspaceEditRequest, + ApplyWorkspaceEditParams, + ApplyWorkspaceEditResponse, + ApplyWorkspaceEditResult, + ), + DOCUMENT_GET: ( + GetDocumentRequest, + GetDocumentParams, + GetDocumentResponse, + GetDocumentResult, + ), + PROJECT_RAW_CONFIG_GET: (GetProjectRawConfigRequest, GetProjectRawConfigParams, GetProjectRawConfigResponse, GetProjectRawConfigResult), + TEXT_DOCUMENT_DID_OPEN: ( + DidOpenTextDocumentNotification, + DidOpenTextDocumentParams, + None, + None, + ), + TEXT_DOCUMENT_DID_CLOSE: ( + DidCloseTextDocumentNotification, + DidCloseTextDocumentParams, + None, + None, + ), +} diff --git a/src/finecode/runner/jsonrpc_client/__init__.py b/src/finecode/runner/jsonrpc_client/__init__.py new file mode 100644 index 0000000..39a7caa --- /dev/null +++ b/src/finecode/runner/jsonrpc_client/__init__.py @@ -0,0 +1,20 @@ +from .client import ( + create_lsp_client_io, + JsonRpcClient, + BaseRunnerRequestException, + NoResponse, + ResponseTimeout, + RunnerFailedToStart, + RequestCancelledError, +) + + +__all__ = [ + "create_lsp_client_io", + "JsonRpcClient", + "BaseRunnerRequestException", + "NoResponse", + "ResponseTimeout", + "RunnerFailedToStart", + "RequestCancelledError", +] diff --git a/src/finecode/runner/jsonrpc_client/_io_thread.py b/src/finecode/runner/jsonrpc_client/_io_thread.py new file mode 100644 index 0000000..1b9157f --- /dev/null +++ b/src/finecode/runner/jsonrpc_client/_io_thread.py @@ -0,0 +1,86 @@ +import asyncio +import threading +import typing +import collections.abc +from loguru import logger + + +class AsyncIOThread: + def __init__(self): + self._thread: threading.Thread | None = None + self._loop: asyncio.AbstractEventLoop | None = None + self._loop_ended_event: typing.Final = threading.Event() + self._running: bool = False + + def start(self) -> None: + if self._running: + raise RuntimeError("IO Thread is already running") + + self._thread = threading.Thread( + target=self._run_loop, name="IO Thread", daemon=True + ) + self._thread.start() + + # Wait for the loop to be ready + while self._loop is None: + threading.Event().wait(0.01) + + self._running = True + logger.debug(f"IO Thread started") + + def stop(self, timeout: float = 5.0) -> None: + if not self._running: + return + + self.run_coroutine(stop_loop_with_timeout(timeout)) + self._running = False + logger.debug("IO Thread stopped") + + def run_coroutine( + self, coro: collections.abc.Coroutine[typing.Any, typing.Any, typing.Any] + ) -> asyncio.Future: + if not self._running or not self._loop: + raise RuntimeError("IO Thread is not running") + + return asyncio.run_coroutine_threadsafe(coro, self._loop) + + @property + def is_running(self) -> bool: + return self._running + + def _run_loop(self) -> None: + try: + self._loop = asyncio.new_event_loop() + asyncio.set_event_loop(self._loop) + + logger.debug(f"IO Thread event loop started") + self._loop.run_forever() + except Exception as e: + logger.error(f"Error in IO Thread event loop: {e}") + finally: + if self._loop and not self._loop.is_closed(): + self._loop.close() + self._loop = None + logger.debug(f"IO Thread event loop stopped") + + +async def stop_loop_with_timeout(timeout: float) -> None: + loop = asyncio.get_running_loop() + tasks = [ + task + for task in asyncio.all_tasks(loop) + if not task.done() and task != asyncio.current_task() + ] + + try: + async with asyncio.timeout(timeout): + await asyncio.gather(*tasks) + except TimeoutError: + logger.debug("Timeout! Cancelling all tasks...") + for task in tasks: + task.cancel() + # Wait for cancellation to complete + await asyncio.gather(*tasks, return_exceptions=True) + + loop.stop() + logger.debug("Stopped event loop in IO thread") diff --git a/src/finecode/runner/jsonrpc_client/client.py b/src/finecode/runner/jsonrpc_client/client.py new file mode 100644 index 0000000..ea53ceb --- /dev/null +++ b/src/finecode/runner/jsonrpc_client/client.py @@ -0,0 +1,1100 @@ +from __future__ import annotations + +import traceback + +import dataclasses +import functools +import os +import shlex +import subprocess +import sys +from pathlib import Path +import asyncio +import json +import re +import threading +import typing +import uuid +import concurrent.futures +import collections.abc + +import culsans +import apischema +from finecode.runner.jsonrpc_client import _io_thread +from loguru import logger + + +class QueueEnd: + # just object() would not support multiprocessing, use class and compare by it + @typing.override + def __eq__(self, other: object) -> bool: + return self.__class__ == other.__class__ + + +QUEUE_END = QueueEnd() + + +# JSON-RPC 2.0 Standard Error Codes +# See: https://www.jsonrpc.org/specification#error_object +class JsonRpcErrorCode: + """Standard JSON-RPC 2.0 error codes.""" + + PARSE_ERROR = -32700 + """Invalid JSON was received by the server. An error occurred on the server while parsing the JSON text.""" + + INVALID_REQUEST = -32600 + """The JSON sent is not a valid Request object.""" + + METHOD_NOT_FOUND = -32601 + """The method does not exist / is not available.""" + + INVALID_PARAMS = -32602 + """Invalid method parameter(s).""" + + INTERNAL_ERROR = -32603 + """Internal JSON-RPC error.""" + + # -32000 to -32099: Server error - Reserved for implementation-defined server-errors + + +class WriterFromQueue: + def __init__(self, out_queue: culsans.SyncQueue[bytes]) -> None: + self._out_queue: typing.Final = out_queue + + def close(self) -> None: + self._out_queue.put(QUEUE_END) + + def write(self, data: bytes) -> None: + self._out_queue.put(data) + + +class RunnerFailedToStart(Exception): + def __init__(self, message: str) -> None: + super().__init__() + self.message: typing.Final = message + + +class BaseRunnerRequestException(Exception): + def __init__(self, message: str) -> None: + super().__init__() + self.message: typing.Final = message + + +class NoResponse(BaseRunnerRequestException): ... + + +class InvalidResponse(BaseRunnerRequestException): ... + + +class ResponseTimeout(BaseRunnerRequestException): ... + + +@dataclasses.dataclass +class ResponseError: + """https://www.jsonrpc.org/specification#error_object""" + + code: int + """A number indicating the error type that occurred.""" + message: str + """A string providing a short description of the error.""" + data: typing.Any | None = None + """A primitive or structured value that contains additional information + about the error. Can be omitted.""" + + +class ErrorOnRequest(BaseRunnerRequestException): + def __init__(self, error: ResponseError) -> None: + super().__init__(message=f"Got error {error.code}: {error.message}") + self.error = error + + +def task_done_log_callback(future: asyncio.Future[typing.Any], task_id: str = ""): + if future.cancelled(): + logger.debug(f"task cancelled: {task_id}") + else: + exc = future.exception() + if exc is not None: + logger.error(f"exception in task: {task_id}") + logger.exception(exc) + else: + logger.trace(f"{task_id} done") + + +class RequestCancelledError(asyncio.CancelledError): + def __init__(self, request_id: int | str) -> None: + super().__init__() + self.request_id = request_id + + +class JsonRpcClient: + CHARSET: typing.Final[str] = "utf-8" + CONTENT_TYPE: typing.Final[str] = "application/vscode-jsonrpc" + VERSION: typing.Final[str] = "2.0" + + def __init__(self, message_types: dict[str, typing.Any], readable_id: str) -> None: + self.server_process_stopped: typing.Final = threading.Event() + self.server_exit_callback: ( + collections.abc.Callable[[], collections.abc.Coroutine] | None + ) = None + self.in_message_queue: typing.Final = culsans.Queue() + self.out_message_queue: typing.Final = culsans.Queue() + self.writer = WriterFromQueue(out_queue=self.out_message_queue.sync_q) + self.message_types = message_types + self.readable_id: str = readable_id + + self._async_tasks: list[asyncio.Task[typing.Any]] = [] + self._stop_event: typing.Final = threading.Event() + self._sync_request_futures: dict[str, concurrent.futures.Future] = {} + self._async_request_futures: dict[str, asyncio.Future] = {} + self._expected_result_type_by_msg_id: dict[str, typing.Any] = {} + + self.feature_impls: dict[str, collections.abc.Callable] = {} + + def feature(self, name: str, impl: collections.abc.Callable) -> None: + self.feature_impls[name] = impl + + async def start_io( + self, cmd: str, io_thread: _io_thread.AsyncIOThread, *args, **kwargs + ): + """Start the given server and communicate with it over stdio.""" + full_cmd = shlex.join([cmd, *args]) + + server_future = io_thread.run_coroutine( + start_server( + full_cmd, + kwargs, + self.in_message_queue, + self.out_message_queue, + request_futures=self._sync_request_futures, + result_types=self._expected_result_type_by_msg_id, + stop_event=self._stop_event, + server_stopped_event=self.server_process_stopped, + server_id=self.readable_id, + ) + ) + + # add done callback to catch exceptions if coroutine fails + server_future.add_done_callback( + functools.partial( + task_done_log_callback, task_id=f"server_future|{self.readable_id}" + ) + ) + + await asyncio.wrap_future(server_future) + server_start_exception = server_future.exception() + if server_start_exception is not None: + # there are no active tasks yet, no need to stop, just interrupt starting + # the server + raise server_start_exception + + message_processor_task = asyncio.create_task(self.process_incoming_messages()) + message_processor_task.add_done_callback( + functools.partial( + task_done_log_callback, + task_id=f"process_incoming_messages|{self.readable_id}", + ) + ) + + notify_exit = asyncio.create_task(self.server_process_stop_handler()) + notify_exit.add_done_callback( + functools.partial( + task_done_log_callback, task_id=f"notify_exit|{self.readable_id}" + ) + ) + + self._async_tasks.extend([message_processor_task, notify_exit]) + logger.debug(f"End of start io for {cmd}") + + async def server_process_stop_handler(self): + """Cleanup handler that runs when the server process managed by the client exits""" + # await asyncio.to_thread(self.server_process_stopped.wait) + + logger.trace(f"Server process stopped handler {self.readable_id}") + while not self.server_process_stopped.is_set(): + await asyncio.sleep(0.1) + + logger.debug(f"Server process {self.readable_id} stopped") + + # Cancel any pending requests + for id_, fut in list(self._sync_request_futures.items()) + list( + self._async_request_futures.items() + ): + if not fut.done(): + fut.set_exception( + RuntimeError("Server was stopped before getting the response") + ) + logger.debug( + f"Cancelled pending request '{id_}': Server was stopped before getting the response" + ) + + if self.server_exit_callback is not None: + await self.server_exit_callback() + + logger.debug(f"End of server stopped handler {self.readable_id}") + + def stop(self) -> None: + self._stop_event.set() + + def _send_data(self, data: str): + header = ( + f"Content-Length: {len(data)}\r\n" + f"Content-Type: {self.CONTENT_TYPE}; charset={self.CHARSET}\r\n\r\n" + ) + data = header + data + + try: + self.writer.write(data.encode(self.CHARSET)) + except Exception as error: + # the writer puts a message in the queue without size, so no exception + # are expected. If one internal come such as shutdown exception because of + # mistake in implementation, log it + logger.error(f"Error sending data to {self.readable_id}:") + logger.exception(error) + + def _send_error_response( + self, + request_id: str | int | None, + code: int, + message: str, + data: typing.Any = None, + ) -> None: + """Send a JSON-RPC error response. + + Args: + request_id: The ID of the request that caused the error. None for notifications. + code: JSON-RPC error code (see JsonRpcErrorCode) + message: Short description of the error + data: Optional additional error information + """ + error_object = {"code": code, "message": message} + if data is not None: + error_object["data"] = data + + response_dict = { + "jsonrpc": self.VERSION, + "id": request_id, + "error": error_object, + } + + response_str = json.dumps(response_dict) + logger.debug(f"Sending error response: {code} - {message}") + self._send_data(response_str) + + def notify(self, method: str, params: typing.Any | None = None) -> None: + logger.debug(f"Sending notification: '{method}' {params}") + + try: + notification_params_type = self.message_types[method][1] + except KeyError: + raise ValueError(f"Type of notification params for {method} not found") + + if notification_params_type is not None: + notification_params_dict = apischema.serialize( + notification_params_type, params, aliaser=apischema.utils.to_camel_case + ) + else: + notification_params_dict = None + + notification_dict = { + "method": method, + "params": notification_params_dict, + "jsonrpc": self.VERSION, + } + + try: + notification_str = json.dumps(notification_dict) + except (TypeError, ValueError) as error: + raise InvalidResponse( + f"Failed to serialize notification: {error}" + ) from error + + self._send_data(notification_str) + + def send_request_sync( + self, + method: str, + params: typing.Any | None = None, + # timeout: float | None = None + ) -> concurrent.futures.Future[typing.Any]: + try: + request_params_type = self.message_types[method][1] + except KeyError: + raise ValueError(f"Type for method {method} not found") + + msg_id = str(uuid.uuid4()) + logger.debug( + f'Sending request with id "{msg_id}": {method} to {self.readable_id}' + ) + + if request_params_type is not None: + request_params_dict = apischema.serialize( + request_params_type, params, aliaser=apischema.utils.to_camel_case + ) + else: + request_params_dict = None + + future = concurrent.futures.Future() + try: + self._expected_result_type_by_msg_id[msg_id] = self.message_types[method][2] + except KeyError: + raise ValueError(f"Message type not found for {method}") + + self._sync_request_futures[msg_id] = future + + request_dict = { + "id": msg_id, + "method": method, + "params": request_params_dict, + "jsonrpc": self.VERSION, + } + + try: + request_str = json.dumps(request_dict) + except (TypeError, ValueError) as error: + # Clean up the future if serialization fails + self._sync_request_futures.pop(msg_id, None) + self._expected_result_type_by_msg_id.pop(msg_id, None) + raise InvalidResponse(f"Failed to serialize request: {error}") from error + + self._send_data(request_str) + + return future + # try: + # response = future.result( + # timeout=timeout, + # ) + # logger.debug(f"Got response on {method} from {self.readable_id}") + # return response + # except TimeoutError: + # raise ResponseTimeout( + # f"Timeout {timeout}s for response on {method} to" + # f" {self.readable_id}" + # ) + + async def send_request( + self, + method: str, + params: typing.Any | None = None, + timeout: float | None = None, + ) -> typing.Any: + try: + request_params_type = self.message_types[method][1] + except KeyError: + raise ValueError(f"Type for method {method} not found") + + msg_id = str(uuid.uuid4()) + logger.debug( + f'Sending request with id "{msg_id}": {method} to {self.readable_id}' + ) + + if request_params_type is not None: + request_params_dict = apischema.serialize( + request_params_type, params, aliaser=apischema.utils.to_camel_case + ) + else: + request_params_dict = None + + message_dict = { + "id": msg_id, + "method": method, + "params": request_params_dict, + "jsonrpc": self.VERSION, + } + + # Serialize request to JSON + try: + request_str = json.dumps(message_dict) + except (TypeError, ValueError) as error: + raise InvalidResponse(f"Failed to serialize request: {error}") from error + + future = asyncio.Future() + self._async_request_futures[msg_id] = future + + try: + self._expected_result_type_by_msg_id[msg_id] = self.message_types[method][2] + except KeyError: + raise ValueError(f"Message type not found for {method}") + + self._send_data(request_str) + + try: + response = await asyncio.wait_for( + future, + timeout, + ) + logger.debug(f"Got response on {method} from {self.readable_id}") + return response + except TimeoutError: + raise ResponseTimeout( + f"Timeout {timeout}s for response on {method} to" + f" runner {self.readable_id}" + ) + except asyncio.CancelledError as error: + raise RequestCancelledError(request_id=msg_id) from error + + async def process_incoming_messages(self) -> None: + logger.debug(f"Start processing messages from server {self.readable_id}") + try: + while not self._stop_event.is_set(): + raw_message = await self.in_message_queue.async_q.get() + if raw_message == QUEUE_END: + logger.debug("Queue with messages from server was closed") + self.in_message_queue.async_q.task_done() + break + + try: + await self.handle_message(raw_message) + except Exception as exc: + logger.exception(exc) + finally: + self.in_message_queue.async_q.task_done() + except asyncio.CancelledError: + ... + + self.in_message_queue.async_q.shutdown() + logger.debug(f"End processing messages from server {self.readable_id}") + + async def handle_message(self, message: dict[str, typing.Any]) -> None: + if "id" in message: + message_id = message["id"] + + if not isinstance(message_id, str) and not isinstance(message_id, int): + logger.warning( + f"Got message with unsupported id: {type(message_id)}, but string or int expected" + ) + return + + if "error" in message: + # error as response + logger.trace(f"Processing message with error on request {message_id}") + # sync request futures are handled in another thread, handle only async + # here + if message_id not in self._async_request_futures: + logger.error( + f"Got error as response for {message_id}, but no response was expected" + ) + return + + future = self._async_request_futures.pop(message_id, None) + if future is None: + logger.error( + f"Got error on request {message_id}, but no response was expected" + ) + return + + try: + response_error = apischema.deserialize( + ResponseError, + data=message["error"], + aliaser=apischema.utils.to_camel_case, + ) + except apischema.ValidationError as error: + exception = InvalidResponse(". ".join(error.messages)) + + # avoid race condition: request is sent, then cancelled and the server + # sends the response before processing the cancel notification + if not future.cancelled(): + future.set_exception(exception) + return + + exception = ErrorOnRequest(error=response_error) + # avoid race condition: request is sent, then cancelled and the server + # sends the response before processing the cancel notification + if not future.cancelled(): + future.set_exception(exception) + return + elif "method" in message: + # incoming request + logger.trace( + f"Processing message with incoming request {message['method']} | {self.readable_id}" + ) + try: + request_type = self.message_types[message["method"]][0] + result_type = self.message_types[message["method"]][3] + except KeyError: + # Method type not registered - send 'Method not found' error + logger.warning( + f"Received request for unregistered method: {message.get('method')} | {self.readable_id}" + ) + self._send_error_response( + request_id=message_id, + code=JsonRpcErrorCode.METHOD_NOT_FOUND, + message="Method not found", + data=f"Method '{message.get('method')}' is not registered", + ) + return + + try: + request = apischema.deserialize( + request_type, message, aliaser=apischema.utils.to_camel_case + ) + except apischema.ValidationError as error: + # Invalid request parameters - send 'Invalid params' error + logger.warning( + f"Invalid params for method {message.get('method')}: {error.messages} | {self.readable_id}" + ) + self._send_error_response( + request_id=message_id, + code=JsonRpcErrorCode.INVALID_PARAMS, + message="Invalid params", + data=". ".join(error.messages), + ) + return + + method = message["method"] + if method not in self.feature_impls: + # Method implementation not found - send 'Method not found' error + logger.warning( + f"Received request for unsupported method: {method} | {self.readable_id}" + ) + self._send_error_response( + request_id=message_id, + code=JsonRpcErrorCode.METHOD_NOT_FOUND, + message="Method not found", + data=f"Method '{method}' is not supported", + ) + return + + impl = self.feature_impls[method] + new_task = asyncio.create_task( + self.run_feature_impl(message_id, impl(request.params), result_type) + ) + self._async_tasks.append(new_task) + else: + # response on our request + logger.trace( + f"Processing message with response to request {message_id}" + ) + if message_id not in self._async_request_futures: + logger.error( + f"Got response to {message_id}, but no response was expected" + ) + return + + # sync request futures are handled in another thread, handle only async + # here + future = self._async_request_futures.pop(message_id, None) + if future is None: + logger.error( + f"Got response to {message_id}, but no response was expected" + ) + return + + result_type = self._expected_result_type_by_msg_id[message_id] + try: + response = apischema.deserialize( + result_type, message, aliaser=apischema.utils.to_camel_case + ) + except apischema.ValidationError as error: + logger.error("errro") + logger.exception(error) + exception = InvalidResponse(". ".join(error.messages)) + + # avoid race condition: request is sent, then cancelled and the server + # sends the response before processing the cancel notification + if not future.cancelled(): + future.set_exception(exception) + return + + # avoid race condition: request is sent, then cancelled and the server + # sends the response before processing the cancel notification + if not future.cancelled(): + future.set_result(response) + logger.trace(f"Successfully processed response to {message_id}") + else: + logger.trace( + f"Request {message_id} was cancelled, ignore the response" + ) + return + else: + # incoming notification + logger.trace( + f"Processing message with incoming notification | {self.readable_id}" + ) + if "method" not in message: + logger.error( + f"Notification expected to have a 'method' field: {message} | {self.readable_id}" + ) + return + + method = message["method"] + if method not in self.feature_impls: + logger.warning( + f"Got notification {method}, but it is not supported | {self.readable_id}" + ) + return + + if method not in self.message_types: + logger.warning(f"Got unsupported notification: {method}") + return + + try: + notification_type = self.message_types[method][0] + notification = apischema.deserialize( + notification_type, message, aliaser=apischema.utils.to_camel_case + ) + except (KeyError, apischema.ValidationError) as error: + logger.warning( + f"Failed to deserialize notification {method}: {error} | {self.readable_id}" + ) + # For notifications, we don't send error responses + return + + impl = self.feature_impls[method] + new_task = asyncio.create_task( + self.run_notification_impl(impl(notification.params)) + ) + self._async_tasks.append(new_task) + + async def run_feature_impl( + self, message_id: str | int, impl_coro, result_type + ) -> None: + try: + result = await impl_coro + + logger.trace(f"{result_type} {result}") + # Send successful response back to the server + response_dict = { + "jsonrpc": self.VERSION, + "id": message_id, + "result": apischema.serialize( + result_type, result, aliaser=apischema.utils.to_camel_case + ), + } + + response_str = json.dumps(response_dict) + logger.debug(f"Sending response for request {message_id}") + self._send_data(response_str) + except Exception as exception: + logger.warning( + f"Error occured on running handler of message {message_id} | {self.readable_id}" + ) + logger.exception(exception) + self._send_error_response( + request_id=message_id, + code=JsonRpcErrorCode.INTERNAL_ERROR, + message="Internal error", + data="", + ) + finally: + current_task = asyncio.current_task() + try: + self._async_tasks.remove(current_task) + except ValueError: + ... + + async def run_notification_impl(self, impl_coro) -> None: + try: + await impl_coro + except Exception as exception: + logger.warning( + f"Error occured on running handler of message | {self.readable_id}" + ) + logger.exception(exception) + finally: + current_task = asyncio.current_task() + try: + self._async_tasks.remove(current_task) + except ValueError: + ... + + +async def start_server( + cmd: str, + subprocess_kwargs: dict[str, str], + in_message_queue: culsans.Queue[bytes], + out_message_queue: culsans.Queue[bytes], + request_futures: dict[str, concurrent.futures.Future[typing.Any]], + result_types: dict[str, typing.Any], + stop_event: threading.Event, + server_stopped_event: threading.Event, + server_id: str, +): + logger.debug(f"Starting server process: {' '.join([cmd, str(subprocess_kwargs)])}") + + server = await asyncio.create_subprocess_shell( + cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + **subprocess_kwargs, + ) + logger.debug(f"{server_id} - process id: {server.pid}") + + tasks: list[asyncio.Task[typing.Any]] = [] + task = asyncio.create_task(log_stderr(server.stderr, stop_event)) + task.add_done_callback( + functools.partial(task_done_log_callback, task_id=f"log_stderr|{server_id}") + ) + tasks.append(task) + + port_future: asyncio.Future[int] = asyncio.Future() + task = asyncio.create_task( + read_stdout(server.stdout, stop_event, port_future, server.pid) + ) + task.add_done_callback( + functools.partial(task_done_log_callback, task_id=f"read_stdout|{server_id}") + ) + tasks.append(task) + + logger.debug(f"Wait for port of {server.pid} | {server_id}") + + try: + await asyncio.wait_for(port_future, 15) + except TimeoutError: + raise RunnerFailedToStart("Didn't get port in 15 seconds") + + port = port_future.result() + logger.debug(f"Got port {port} of {server.pid} | {server_id}") + + try: + reader, writer = await asyncio.open_connection("127.0.0.1", port) + except Exception as exc: + logger.exception(exc) + + for task in tasks: + task.cancel() + + raise exc + + task = asyncio.create_task( + read_messages_from_reader( + reader, + in_message_queue.sync_q, + request_futures, + result_types, + stop_event, + server.pid, + ) + ) + task.add_done_callback( + functools.partial( + task_done_log_callback, task_id=f"read_messages_from_reader|{server_id}" + ) + ) + tasks.append(task) + + task = asyncio.create_task( + send_messages_from_queue(queue=out_message_queue.async_q, writer=writer) + ) + task.add_done_callback( + functools.partial( + task_done_log_callback, task_id=f"send_messages_from_queue|{server_id}" + ) + ) + tasks.append(task) + + task = asyncio.create_task( + wait_for_stop_event_and_clean( + stop_event, server, tasks, server_stopped_event, out_message_queue.async_q + ) + ) + task.add_done_callback( + functools.partial( + task_done_log_callback, task_id=f"wait_for_stop_event_and_clean|{server_id}" + ) + ) + + logger.debug(f"Server {server.pid} started | {server_id}") + + +async def wait_for_stop_event_and_clean( + stop_event: threading.Event, + server_process: asyncio.subprocess.Process, + tasks: list[asyncio.Task[typing.Any]], + server_stopped_event: threading.Event, + out_message_queue: culsans.AsyncQueue[bytes], +) -> None: + # wait either on stop event (=user asks to stop the client) or end of the server + # process + logger.debug("Wait on one of tasks") + tasks_to_wait = [ + asyncio.create_task(asyncio.to_thread(stop_event.wait)), + asyncio.create_task(server_process.wait()), + ] + _, _ = await asyncio.wait(tasks_to_wait, return_when=asyncio.FIRST_COMPLETED) + logger.debug("One of tasks to wait is done") + + if not stop_event.is_set(): + stop_event.set() + + # close the WriterFromQueue + await out_message_queue.put(QUEUE_END) + + if server_process.returncode is None: + logger.debug("Wait for the end of server process") + _ = await server_process.wait() + logger.debug(f"Server process ended with code {server_process.returncode}") + + for task in tasks: + if not task.done(): + task.cancel() + + server_stopped_event.set() + logger.debug("Cleaned resources of client") + + +async def log_stderr(stderr: asyncio.StreamReader, stop_event: threading.Event) -> None: + """Read and log stderr output from the subprocess.""" + logger.debug("Start reading logs from stderr") + try: + while not stop_event.is_set(): + line = await stderr.readline() + if not line: + break + logger.debug( + f"Server stderr: {line.decode('utf-8', errors='replace').rstrip()}" + ) + except asyncio.CancelledError: + pass + + logger.debug("End reading logs from stderr") + + +async def read_stdout( + stdout: asyncio.StreamReader, + stop_event: threading.Event, + port_future: asyncio.Future[int], + server_pid: int, +) -> None: + logger.debug(f"Start reading logs from stdout | {server_pid}") + try: + while not stop_event.is_set(): + try: + line = await stdout.readline() + except ValueError as exception: + logger.error(exception) + continue + + if not line: + break + if b"Serving on (" in line: + match = re.search(rb"Serving on \('[\d.]+', (\d+)\)", line) + if match: + port = int(match.group(1)) + port_future.set_result(port) + # logger.debug( + # f"Server {server_pid} stdout: {line.decode('utf-8', errors='replace').rstrip()}" + # ) + except asyncio.CancelledError: + pass + + logger.debug(f"End reading logs from stdout | {server_pid}") + + +async def send_messages_from_queue( + queue: culsans.AsyncQueue[bytes], writer: asyncio.StreamWriter +) -> None: + logger.debug("Start sending messages from queue") + + try: + while True: + message = await queue.get() + if message == QUEUE_END: + writer.close() + logger.debug("Queue was closed, stop sending") + break + writer.write(message) + await writer.drain() + except asyncio.CancelledError: + ... + + queue.shutdown() + logger.debug("End sending messages from queue") + + +CONTENT_LENGTH_PATTERN = re.compile(rb"^Content-Length: (\d+)\r\n$") + + +async def read_messages_from_reader( + reader: asyncio.StreamReader, + message_queue: culsans.SyncQueue[bytes], + request_futures: dict[str, concurrent.futures.Future[typing.Any]], + result_types: dict[str, typing.Any], + stop_event: threading.Event, + server_pid: int, +) -> None: + content_length = 0 + + try: + while not stop_event.is_set(): + try: + try: + header = await reader.readline() + except ValueError: + logger.error(f"Value error in readline of {server_pid}") + continue + except ConnectionResetError: + logger.warning( + f"Server {server_pid} closed the connection(ConnectionResetError), stop the client" + ) + stop_event.set() + break + + if not header: + if reader.at_eof(): + logger.debug(f"Reader reached EOF | {server_pid}") + break + continue + + # Extract content length if possible + if not content_length: + match = CONTENT_LENGTH_PATTERN.fullmatch(header) + if match: + content_length = int(match.group(1)) + logger.debug(f"Content length | {server_pid}: {content_length}") + else: + logger.debug( + f"Not matched content length: {header} | {server_pid}" + ) + + # Check if all headers have been read (as indicated by an empty line \r\n) + if content_length and not header.strip(): + # Read body + body = None + try: + body = await reader.readexactly(content_length) + except asyncio.IncompleteReadError as error: + logger.debug( + f"Incomplete read error: {error} | {server_pid} : {error.partial}" + ) + content_length = 0 + continue + except ConnectionResetError: + logger.warning( + f"Server {server_pid} closed the connection(ConnectionResetError), stop the client" + ) + stop_event.set() + break + + if not body: + content_length = 0 + continue + + logger.debug(f"Got content {server_pid}: {body}") + try: + message = json.loads(body) + except json.JSONDecodeError as exc: + logger.error( + f"Failed to parse JSON message: {exc} | {server_pid}" + ) + continue + finally: + # Reset + content_length = 0 + + if not isinstance(message, dict): + logger.error("JSON Message expected to be a dict") + continue + if "jsonrpc" not in message: + logger.error("JSON Message expected to contain 'jsonrpc' key") + continue + + if message["jsonrpc"] != JsonRpcClient.VERSION: + logger.warning(f'Unknown message "{message}" | {server_pid}') + continue + + # error should be also handled here + is_response = ( + "id" in message + and "error" not in message + and "method" not in message + ) + + if is_response: + logger.debug(f"Response message received. | {server_pid}") + msg_id = message["id"] + raw_result = message.get("result", None) + future = request_futures.pop(msg_id, None) + + if future is not None: + try: + result_type = result_types[msg_id] + except KeyError: + logger.error( + f"Result type not found for message {msg_id}" + ) + continue + + try: + result = apischema.deserialize( + result_type, + raw_result, + aliaser=apischema.utils.to_camel_case, + ) + except apischema.ValidationError as error: + exception = InvalidResponse(". ".join(error.messages)) + if not future.cancelled(): + future.set_exception(exception) + continue + + logger.debug( + f'Received result for message "{msg_id}" | {server_pid}' + ) + if not future.cancelled(): + future.set_result(result) + else: + message_queue.put(message) + else: + # incoming request or notification + message_queue.put(message) + else: + if not header.startswith( + b"Content-Length:" + ) and not header.startswith(b"Content-Type:"): + logger.debug( + f'Something is wrong: {content_length} "{header}" {not header.strip()} | {server_pid}' + ) + except Exception as exc: + logger.exception( + f"Exception in message reader loop | {server_pid}: {exc}" + ) + # Reset state to avoid infinite loop on persistent errors + content_length = 0 + except asyncio.CancelledError: + ... + + logger.debug(f"End reading messages from reader | {server_pid}") + + +async def create_lsp_client_io( + server_cmd: str, + working_dir_path: Path, + message_types: dict[str, typing.Any], + io_thread: _io_thread.AsyncIOThread, + readable_id: str, +) -> JsonRpcClient: + ls = JsonRpcClient(message_types=message_types, readable_id=readable_id) + splitted_cmd = shlex.split(server_cmd) + executable, *args = splitted_cmd + + old_working_dir = os.getcwd() + os.chdir(working_dir_path) + + # temporary remove VIRTUAL_ENV env variable to avoid starting in wrong venv + old_virtual_env_var = os.environ.pop("VIRTUAL_ENV", None) + + creationflags = 0 + # start_new_session = True .. process has parent id of real parent, but is not + # ended if parent was ended + start_new_session = True + if sys.platform == "win32": + # use creationflags because `start_new_session` doesn't work on Windows + # subprocess.CREATE_NO_WINDOW .. no console window on Windows. TODO: test + creationflags = subprocess.DETACHED_PROCESS | subprocess.CREATE_NO_WINDOW + start_new_session = False + + await ls.start_io( + executable, + io_thread, + *args, + start_new_session=start_new_session, + creationflags=creationflags, + ) + if old_virtual_env_var is not None: + os.environ["VIRTUAL_ENV"] = old_virtual_env_var + + os.chdir(old_working_dir) # restore original working directory + return ls + + +__all__ = ["create_lsp_client_io", "JsonRpcClient"] diff --git a/src/finecode/runner/runner_client.py b/src/finecode/runner/runner_client.py index a81d791..11f88c0 100644 --- a/src/finecode/runner/runner_client.py +++ b/src/finecode/runner/runner_client.py @@ -1,150 +1,60 @@ -# TODO: pass not the whole runner, but only runner.client -# TODO: autocheck, that runner.client.protocol is accessed only here -# TODO: autocheck, that lsprotocol is imported only here +""" +API of ER client for "higher" layers like services, CLI. +""" + from __future__ import annotations import asyncio -import asyncio.subprocess import dataclasses import enum import json import typing import pathlib -from typing import TYPE_CHECKING, Any +from typing import Any from loguru import logger -from lsprotocol import types -from pygls import exceptions as pygls_exceptions import finecode.domain as domain +from finecode.runner import jsonrpc_client, _internal_client_types, _internal_client_api -if TYPE_CHECKING: - from finecode.runner.runner_info import ExtensionRunnerInfo - - -class BaseRunnerRequestException(Exception): - def __init__(self, message: str) -> None: - self.message = message - - -class NoResponse(BaseRunnerRequestException): ... - - -class ResponseTimeout(BaseRunnerRequestException): ... +# reexport +BaseRunnerRequestException = jsonrpc_client.BaseRunnerRequestException +GetDocumentParams = _internal_client_types.GetDocumentParams +GetDocumentResult = _internal_client_types.GetDocumentResult -class ActionRunFailed(BaseRunnerRequestException): ... +class ActionRunFailed(jsonrpc_client.BaseRunnerRequestException): ... -class ActionRunStopped(BaseRunnerRequestException): ... - -async def log_process_log_streams(process: asyncio.subprocess.Process) -> None: - stdout, stderr = await process.communicate() - - logger.info(f"[Runner exited with {process.returncode}]") - if stdout: - logger.info(f"[stdout]\n{stdout.decode()}") - if stderr: - logger.error(f"[stderr]\n{stderr.decode()}") - - -async def send_request( - runner: ExtensionRunnerInfo, - method: str, - params: Any | None, - timeout: int | None = 10, -) -> Any: - logger.debug(f"Send {method} to {runner.readable_id}") - try: - response = await asyncio.wait_for( - runner.client.protocol.send_request_async( - method=method, - params=params, - ), - timeout, - ) - logger.debug(f"Got response on {method} from {runner.readable_id}") - return response - except RuntimeError as error: - logger.error(f"Extension runner crashed: {error}") - await log_process_log_streams(process=runner.client._server) - raise NoResponse( - f"Extension runner {runner.readable_id} crashed, no response on {method}" - ) - except TimeoutError: - raise ResponseTimeout( - f"Timeout {timeout}s for response on {method} to" - f" runner {runner.readable_id}" - ) - except pygls_exceptions.JsonRpcInternalError as error: - logger.error(f"JsonRpcInternalError: {error.message} {error.data}") - raise NoResponse( - f"Extension runner {runner.readable_id} returned no response," - f" check it logs: {runner.logs_path}" - ) +class ActionRunStopped(jsonrpc_client.BaseRunnerRequestException): ... -def send_request_sync( - runner: ExtensionRunnerInfo, - method: str, - params: Any | None, - timeout: int | None = 10, -) -> Any | None: - try: - response_future = runner.client.protocol.send_request( - method=method, - params=params, - ) - response = response_future.result(timeout) - logger.debug(f"Got response on {method} from {runner.readable_id}") - return response - except RuntimeError as error: - logger.error(f"Extension runner crashed? {error}") - raise NoResponse( - f"Extension runner {runner.readable_id} crashed, no response on {method}" - ) - except TimeoutError: - if runner.client._server.returncode is not None: - logger.error( - "Extension runner stopped with" - f" exit code {runner.client._server.returncode}" - ) - raise ResponseTimeout( - f"Timeout {timeout}s for response on {method}" - f" to runner {runner.readable_id}" - ) - except pygls_exceptions.JsonRpcInternalError as error: - logger.error(f"JsonRpcInternalError: {error.message} {error.data}") - raise NoResponse( - f"Extension runner {runner.readable_id} returned no response," - f" check it logs: {runner.logs_path}" - ) +@dataclasses.dataclass +class ExtensionRunnerInfo: + working_dir_path: pathlib.Path + env_name: str + status: RunnerStatus + # NOTE: initialized doesn't mean the runner is running, check its status + initialized_event: asyncio.Event + # e.g. if there is no venv for env, client can be None + client: jsonrpc_client.JsonRpcClient | None = None + @property + def readable_id(self) -> str: + return f"{self.working_dir_path} ({self.env_name})" -async def initialize( - runner: ExtensionRunnerInfo, - client_process_id, - client_name: str, - client_version: str, -) -> None: - await send_request( - runner=runner, - method=types.INITIALIZE, - params=types.InitializeParams( - process_id=client_process_id, - capabilities=types.ClientCapabilities(), - client_info=types.ClientInfo(name=client_name, version=client_version), - trace=types.TraceValue.Verbose, - ), - timeout=20, - ) + @property + def logs_path(self) -> pathlib.Path: + return self.working_dir_path / ".venvs" / self.env_name / "logs" / "runner.log" -async def notify_initialized(runner: ExtensionRunnerInfo) -> None: - runner.client.protocol.notify( - method=types.INITIALIZED, params=types.InitializedParams() - ) +class RunnerStatus(enum.Enum): + NO_VENV = enum.auto() + INITIALIZING = enum.auto() + FAILED = enum.auto() + RUNNING = enum.auto() + EXITED = enum.auto() # JSON object or text @@ -170,34 +80,52 @@ async def run_action( if not runner.initialized_event.is_set(): await runner.initialized_event.wait() - response = await send_request( - runner=runner, - method=types.WORKSPACE_EXECUTE_COMMAND, - params=types.ExecuteCommandParams( - command="actions/run", - arguments=[action_name, params, options], - ), - timeout=None, - ) + if runner.status != RunnerStatus.RUNNING: + raise ActionRunFailed( + f"Runner {runner.readable_id} is not running: {runner.status}" + ) + + try: + response = await runner.client.send_request( + method=_internal_client_types.WORKSPACE_EXECUTE_COMMAND, + params=_internal_client_types.ExecuteCommandParams( + command="actions/run", + arguments=[action_name, params, options], + ), + timeout=None, + ) + except jsonrpc_client.RequestCancelledError as error: + logger.trace( + f"Request {error.request_id} to {runner.readable_id} was cancelled" + ) + await _internal_client_api.cancel_request( + client=runner.client, request_id=error.request_id + ) + raise error + + command_result = response.result - if hasattr(response, "error"): - raise ActionRunFailed(response.error) + if "error" in command_result: + raise ActionRunFailed(command_result["error"]) - return_code = response.return_code + return_code = command_result["return_code"] raw_result = "" - stringified_result = response.result + stringified_result = command_result["result"] # currently result is always dumped to json even if response format is expected to # be a string. See docs of ER lsp server for more details. raw_result = json.loads(stringified_result) - if response.format == "string": + if command_result["format"] == "string": result = raw_result - elif response.format == "json" or response.format == "styled_text_json": + elif ( + command_result["format"] == "json" + or command_result["format"] == "styled_text_json" + ): # string was already converted to dict above result = raw_result else: - raise Exception(f"Not support result format: {response.format}") + raise Exception(f"Not support result format: {command_result['format']}") - if response.status == "stopped": + if command_result["status"] == "stopped": raise ActionRunStopped(message=result) return RunActionResponse(result=result, return_code=return_code) @@ -207,10 +135,9 @@ async def reload_action(runner: ExtensionRunnerInfo, action_name: str) -> None: if not runner.initialized_event.is_set(): await runner.initialized_event.wait() - await send_request( - runner=runner, - method=types.WORKSPACE_EXECUTE_COMMAND, - params=types.ExecuteCommandParams( + await runner.client.send_request( + method=_internal_client_types.WORKSPACE_EXECUTE_COMMAND, + params=_internal_client_types.ExecuteCommandParams( command="actions/reload", arguments=[ action_name, @@ -226,17 +153,16 @@ async def resolve_package_path( # config, which is then registered in runner. In this time runner is not available # for any other actions, so `runner.started_event` stays not set and should not be # checked here. - response = await send_request( - runner=runner, - method=types.WORKSPACE_EXECUTE_COMMAND, - params=types.ExecuteCommandParams( + response = await runner.client.send_request( + method=_internal_client_types.WORKSPACE_EXECUTE_COMMAND, + params=_internal_client_types.ExecuteCommandParams( command="packages/resolvePath", arguments=[ package_name, ], ), ) - return {"packagePath": response.packagePath} + return {"packagePath": response.result["packagePath"]} @dataclasses.dataclass @@ -245,52 +171,37 @@ class RunnerConfig: # config by handler source action_handler_configs: dict[str, dict[str, Any]] + def to_dict(self) -> dict[str, typing.Any]: + return { + "actions": [action.to_dict() for action in self.actions], + "action_handler_configs": self.action_handler_configs, + } + async def update_config( runner: ExtensionRunnerInfo, project_def_path: pathlib.Path, config: RunnerConfig ) -> None: - await send_request( - runner=runner, - method=types.WORKSPACE_EXECUTE_COMMAND, - params=types.ExecuteCommandParams( + await runner.client.send_request( + method=_internal_client_types.WORKSPACE_EXECUTE_COMMAND, + params=_internal_client_types.ExecuteCommandParams( command="finecodeRunner/updateConfig", arguments=[ runner.working_dir_path.as_posix(), runner.working_dir_path.stem, project_def_path.as_posix(), - config, + config.to_dict(), ], ), ) -async def shutdown( - runner: ExtensionRunnerInfo, -) -> None: - await send_request(runner=runner, method=types.SHUTDOWN, params=None) - - -def shutdown_sync( - runner: ExtensionRunnerInfo, -) -> None: - send_request_sync(runner=runner, method=types.SHUTDOWN, params=None) - - -async def exit(runner: ExtensionRunnerInfo) -> None: - runner.client.protocol.notify(method=types.EXIT) - - -def exit_sync(runner: ExtensionRunnerInfo) -> None: - runner.client.protocol.notify(method=types.EXIT) - - async def notify_document_did_open( runner: ExtensionRunnerInfo, document_info: domain.TextDocumentInfo ) -> None: - runner.client.protocol.notify( - method=types.TEXT_DOCUMENT_DID_OPEN, - params=types.DidOpenTextDocumentParams( - text_document=types.TextDocumentItem( + runner.client.notify( + method=_internal_client_types.TEXT_DOCUMENT_DID_OPEN, + params=_internal_client_types.DidOpenTextDocumentParams( + text_document=_internal_client_types.TextDocumentItem( uri=document_info.uri, language_id="", version=int(document_info.version), @@ -303,9 +214,27 @@ async def notify_document_did_open( async def notify_document_did_close( runner: ExtensionRunnerInfo, document_uri: str ) -> None: - runner.client.protocol.notify( - method=types.TEXT_DOCUMENT_DID_CLOSE, - params=types.DidCloseTextDocumentParams( - text_document=types.TextDocumentIdentifier(document_uri) + runner.client.notify( + method=_internal_client_types.TEXT_DOCUMENT_DID_CLOSE, + params=_internal_client_types.DidCloseTextDocumentParams( + text_document=_internal_client_types.TextDocumentIdentifier(document_uri) ), ) + + +__all__ = [ + "ActionRunFailed", + "ActionRunStopped", + "ExtensionRunnerInfo", + "RunnerStatus", + "RunActionRawResult", + "RunActionResponse", + "RunResultFormat", + "run_action", + "reload_action", + "resolve_package_path", + "RunnerConfig", + "update_config", + "notify_document_did_open", + "notify_document_did_close", +] diff --git a/src/finecode/runner/runner_info.py b/src/finecode/runner/runner_info.py deleted file mode 100644 index 79ac29e..0000000 --- a/src/finecode/runner/runner_info.py +++ /dev/null @@ -1,100 +0,0 @@ -from __future__ import annotations - -import asyncio -import enum -import logging -import shlex -from dataclasses import dataclass -from pathlib import Path -from typing import Callable, Coroutine - -from pygls.io_ import run_async - -from finecode.pygls_client_utils import JsonRPCClient - - -class CustomJsonRpcClient(JsonRPCClient): - def __init__(self, *args, **kwargs) -> None: - super().__init__(*args, **kwargs) - self.server_exit_callback: Callable[[], Coroutine] | None = None - - async def start_io(self, cmd: str, *args, **kwargs): - """Start the given server and communicate with it over stdio.""" - logger = logging.getLogger(__name__) - logger.debug("Starting server process: %s", " ".join([cmd, *args, str(kwargs)])) - # difference with original version: use `create_subprocess_shell` instead of - # `create_subprocess_exec` to be able to start detached process - full_cmd = shlex.join([cmd, *args]) - server = await asyncio.create_subprocess_shell( - full_cmd, - stdout=asyncio.subprocess.PIPE, - stdin=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - **kwargs, - ) - logger.debug(f"{cmd} - process id: {server.pid}") - - # Keep mypy happy - if server.stdout is None: - raise RuntimeError("Server process is missing a stdout stream") - - # Keep mypy happy - if server.stdin is None: - raise RuntimeError("Server process is missing a stdin stream") - - self.protocol.set_writer(server.stdin) - connection = asyncio.create_task( - run_async( - stop_event=self._stop_event, - reader=server.stdout, - protocol=self.protocol, - logger=logger, - error_handler=self.report_server_error, - ) - ) - notify_exit = asyncio.create_task(self._server_exit()) - - self._server = server - self._async_tasks.extend([connection, notify_exit]) - - async def server_exit(self, server): - result = await super().server_exit(server) - if self.server_exit_callback is not None: - await self.server_exit_callback() - return result - - -@dataclass -class ExtensionRunnerInfo: - working_dir_path: Path - env_name: str - status: RunnerStatus - # NOTE: initialized doesn't mean the runner is running, check its status - initialized_event: asyncio.Event - # e.g. if there is no venv for env, client can be None - client: CustomJsonRpcClient | None = None - keep_running_request_task: asyncio.Task | None = None - - @property - def process_id(self) -> int: - if self.client is not None and self.client._server is not None: - return self.client._server.pid - else: - return 0 - - @property - def readable_id(self) -> str: - return f"{self.working_dir_path} ({self.env_name})" - - @property - def logs_path(self) -> Path: - return self.working_dir_path / ".venvs" / self.env_name / "logs" / "runner.log" - - -class RunnerStatus(enum.Enum): - READY_TO_START = enum.auto() - NO_VENV = enum.auto() - INITIALIZING = enum.auto() - FAILED = enum.auto() - RUNNING = enum.auto() - EXITED = enum.auto() diff --git a/src/finecode/runner/manager.py b/src/finecode/runner/runner_manager.py similarity index 60% rename from src/finecode/runner/manager.py rename to src/finecode/runner/runner_manager.py index 29b9739..2701b64 100644 --- a/src/finecode/runner/manager.py +++ b/src/finecode/runner/runner_manager.py @@ -1,32 +1,40 @@ +""" +API to manage ERs: start, stop, restart. +""" + import asyncio +import collections.abc import json import os import shutil from pathlib import Path -from typing import Callable, Coroutine +import typing from loguru import logger -from lsprotocol import types from finecode import context, domain, finecode_cmd from finecode.config import collect_actions, config_models, read_configs -from finecode.pygls_client_utils import create_lsp_client_io -from finecode.runner import runner_client, runner_info +from finecode.runner import ( + jsonrpc_client, + runner_client, + _internal_client_api, + _internal_client_types, +) +from finecode.runner.jsonrpc_client import _io_thread from finecode.utils import iterable_subscribe project_changed_callback: ( - Callable[[domain.Project], Coroutine[None, None, None]] | None + typing.Callable[[domain.Project], collections.abc.Coroutine[None, None, None]] + | None ) = None -get_document: Callable[[], Coroutine] | None = None -apply_workspace_edit: Callable[[], Coroutine] | None = None +get_document: typing.Callable[[], collections.abc.Coroutine] | None = None +apply_workspace_edit: typing.Callable[[], collections.abc.Coroutine] | None = None partial_results: iterable_subscribe.IterableSubscribe = ( iterable_subscribe.IterableSubscribe() ) - -class RunnerFailedToStart(Exception): - def __init__(self, message: str) -> None: - self.message = message +# reexport +RunnerFailedToStart = jsonrpc_client.RunnerFailedToStart async def notify_project_changed(project: domain.Project) -> None: @@ -34,30 +42,33 @@ async def notify_project_changed(project: domain.Project) -> None: await project_changed_callback(project) -async def _apply_workspace_edit(params: types.ApplyWorkspaceEditParams): +async def _apply_workspace_edit( + params: _internal_client_types.ApplyWorkspaceEditParams, +): def map_change_object(change): - return types.TextEdit( - range=types.Range( - start=types.Position( + return _internal_client_types.TextEdit( + range=_internal_client_types.Range( + start=_internal_client_types.Position( line=change.range.start.line, character=change.range.start.character ), - end=types.Position( + end=_internal_client_types.Position( change.range.end.line, character=change.range.end.character ), ), new_text=change.newText, ) - converted_params = types.ApplyWorkspaceEditParams( - edit=types.WorkspaceEdit( + converted_params = _internal_client_types.ApplyWorkspaceEditParams( + edit=_internal_client_types.WorkspaceEdit( document_changes=[ - types.TextDocumentEdit( - text_document=types.OptionalVersionedTextDocumentIdentifier( - document_edit.textDocument.uri + _internal_client_types.TextDocumentEdit( + text_document=_internal_client_types.OptionalVersionedTextDocumentIdentifier( + document_edit.text_document.uri ), edits=[map_change_object(change) for change in document_edit.edits], ) - for document_edit in params.edit.documentChanges + for document_edit in params.edit.document_changes + if isinstance(document_edit, _internal_client_types.TextDocumentEdit) ] ) ) @@ -65,35 +76,41 @@ def map_change_object(change): async def _start_extension_runner_process( - runner_dir: Path, env_name: str, ws_context: context.WorkspaceContext -) -> runner_info.ExtensionRunnerInfo | None: - runner_info_instance = runner_info.ExtensionRunnerInfo( - working_dir_path=runner_dir, - env_name=env_name, - status=runner_info.RunnerStatus.READY_TO_START, - initialized_event=asyncio.Event(), - client=None, - ) - + runner: runner_client.ExtensionRunnerInfo, ws_context: context.WorkspaceContext +) -> None: try: - python_cmd = finecode_cmd.get_python_cmd(runner_dir, env_name) + python_cmd = finecode_cmd.get_python_cmd( + runner.working_dir_path, runner.env_name + ) except ValueError: try: - runner_info_instance.status = runner_info.RunnerStatus.NO_VENV - await notify_project_changed(ws_context.ws_projects[runner_dir]) + runner.status = runner_client.RunnerStatus.NO_VENV + await notify_project_changed( + ws_context.ws_projects[runner.working_dir_path] + ) except KeyError: ... logger.error( - f"Project {runner_dir} uses finecode, but env (venv) doesn't exist yet. Run `prepare_env` command to create it" + f"Project {runner.working_dir_path} uses finecode, but env (venv) doesn't exist yet. Run `prepare_env` command to create it" + ) + + raise jsonrpc_client.RunnerFailedToStart( + f"Runner '{runner.readable_id}' failed to start" ) - return None + + if ws_context.runner_io_thread is None: + logger.trace("Starting IO Thread") + ws_context.runner_io_thread = _io_thread.AsyncIOThread() + ws_context.runner_io_thread.start() process_args: list[str] = [ "--trace", - f"--project-path={runner_dir.as_posix()}", - f"--env-name={env_name}", + f"--project-path={runner.working_dir_path.as_posix()}", + f"--env-name={runner.env_name}", + ] + env_config = ws_context.ws_projects[runner.working_dir_path].env_configs[ + runner.env_name ] - env_config = ws_context.ws_projects[runner_dir].env_configs[env_name] runner_config = env_config.runner_config # TODO: also check whether lsp server is available, without it doesn't make sense # to start with debugger @@ -103,46 +120,50 @@ async def _start_extension_runner_process( process_args.append("--debug-port=5681") process_args_str: str = " ".join(process_args) - client = await create_lsp_client_io( - runner_info.CustomJsonRpcClient, + client = await jsonrpc_client.create_lsp_client_io( f"{python_cmd} -m finecode_extension_runner.cli start {process_args_str}", - runner_dir, + runner.working_dir_path, + message_types=_internal_client_types.METHOD_TO_TYPES, + io_thread=ws_context.runner_io_thread, + readable_id=runner.readable_id, ) - runner_info_instance.client = client + runner.client = client # TODO: recognize started debugger and send command to lsp server async def on_exit(): - logger.debug(f"Extension Runner {runner_info_instance.readable_id} exited") - runner_info_instance.status = runner_info.RunnerStatus.EXITED - await notify_project_changed(ws_context.ws_projects[runner_dir]) # TODO: fix + logger.debug(f"Extension Runner {runner.readable_id} exited") + runner.status = runner_client.RunnerStatus.EXITED + await notify_project_changed( + ws_context.ws_projects[runner.working_dir_path] + ) # TODO: fix # TODO: restart if WM is not stopping - runner_info_instance.client.server_exit_callback = on_exit + runner.client.server_exit_callback = on_exit if get_document is not None: - register_get_document_feature = runner_info_instance.client.feature( - "documents/get" + runner.client.feature( + _internal_client_types.DOCUMENT_GET, + get_document, ) - register_get_document_feature(get_document) - register_workspace_apply_edit = runner_info_instance.client.feature( - types.WORKSPACE_APPLY_EDIT + runner.client.feature( + _internal_client_types.WORKSPACE_APPLY_EDIT, _apply_workspace_edit ) - register_workspace_apply_edit(_apply_workspace_edit) - async def on_progress(params: types.ProgressParams): + async def on_progress(params: _internal_client_types.ProgressParams): logger.debug(f"Got progress from runner for token: {params.token}") partial_result = domain.PartialResult( token=params.token, value=json.loads(params.value) ) partial_results.publish(partial_result) - register_progress_feature = runner_info_instance.client.feature(types.PROGRESS) - register_progress_feature(on_progress) + runner.client.feature(_internal_client_types.PROGRESS, on_progress) - async def get_project_raw_config(params): + async def get_project_raw_config( + params: _internal_client_types.GetProjectRawConfigParams, + ): logger.debug(f"Get project raw config: {params}") - project_def_path_str = params.projectDefPath + project_def_path_str = params.project_def_path project_def_path = Path(project_def_path_str) try: project_raw_config = ws_context.ws_projects_raw_configs[ @@ -150,52 +171,42 @@ async def get_project_raw_config(params): ] except KeyError: raise ValueError(f"Config of project '{project_def_path_str}' not found") - return {"config": json.dumps(project_raw_config)} + return _internal_client_types.GetProjectRawConfigResult( + config=json.dumps(project_raw_config) + ) - register_get_project_raw_config_feature = runner_info_instance.client.feature( - "projects/getRawConfig" + runner.client.feature( + _internal_client_types.PROJECT_RAW_CONFIG_GET, + get_project_raw_config, ) - register_get_project_raw_config_feature(get_project_raw_config) - return runner_info_instance - -async def stop_extension_runner(runner: runner_info.ExtensionRunnerInfo) -> None: +async def stop_extension_runner(runner: runner_client.ExtensionRunnerInfo) -> None: logger.trace(f"Trying to stop extension runner {runner.readable_id}") - if not runner.client.stopped: - logger.debug("Send shutdown to server") + if runner.status == runner_client.RunnerStatus.RUNNING: try: - await runner_client.shutdown(runner=runner) + await _internal_client_api.shutdown(client=runner.client) except Exception as e: - logger.error(f"Failed to shutdown: {e}") + logger.error(f"Failed to shutdown:") + logger.exception(e) - await runner_client.exit(runner) - logger.debug("Sent exit to server") - await runner.client.stop() - logger.trace( - f"Stop extension runner {runner.process_id} in {runner.readable_id}" - ) + await _internal_client_api.exit(client=runner.client) + logger.trace(f"Stopped extension runner {runner.readable_id}") else: logger.trace("Extension runner was not running") -def stop_extension_runner_sync(runner: runner_info.ExtensionRunnerInfo) -> None: +def stop_extension_runner_sync(runner: runner_client.ExtensionRunnerInfo) -> None: logger.trace(f"Trying to stop extension runner {runner.readable_id}") - if not runner.client.stopped: - logger.debug("Send shutdown to server") + if runner.status == runner_client.RunnerStatus.RUNNING: try: - runner_client.shutdown_sync(runner=runner) - except Exception: - # currently we get (almost?) always this error. TODO: Investigate why - # mute for now to make output less verbose - # logger.error(f"Failed to shutdown: {e}") - ... + _internal_client_api.shutdown_sync(client=runner.client) + except Exception as e: + logger.error(f"Failed to shutdown:") + logger.exception(e) - runner_client.exit_sync(runner) - logger.debug("Sent exit to server") - logger.trace( - f"Stop extension runner {runner.process_id} in {runner.readable_id}" - ) + _internal_client_api.exit_sync(runner.client) + logger.trace(f"Stopped extension runner {runner.readable_id}") else: logger.trace("Extension runner was not running") @@ -212,14 +223,26 @@ async def start_runners_with_presets( for project in projects: project_status = project.status if project_status == domain.ProjectStatus.CONFIG_VALID: - task = tg.create_task( - _start_dev_workspace_runner( - project_def=project, ws_context=ws_context + # first check whether runner doesn't exist yet to avoid duplicates + project_runners = ws_context.ws_projects_extension_runners.get(project.dir_path, {}) + project_dev_workspace_runner = project_runners.get('dev_workspace', None) + start_new_runner = True + if project_dev_workspace_runner is not None and project_dev_workspace_runner.status in [runner_client.RunnerStatus.INITIALIZING, runner_client.RunnerStatus.RUNNING]: + # start a new one only if: + # - either there is no runner yet + # or venv exist(=exclude `runner_client.RunnerStatus.NO_VENV`) + # and runner is not initializing or running already + start_new_runner = False + + if start_new_runner: + task = tg.create_task( + _start_dev_workspace_runner( + project_def=project, ws_context=ws_context + ) ) - ) - new_runners_tasks.append(task) + new_runners_tasks.append(task) elif project_status != domain.ProjectStatus.NO_FINECODE: - raise RunnerFailedToStart( + raise jsonrpc_client.RunnerFailedToStart( f"Project '{project.name}' has invalid configuration, status: {project_status.name}" ) @@ -229,18 +252,18 @@ async def start_runners_with_presets( except ExceptionGroup as eg: for exception in eg.exceptions: if isinstance( - exception, runner_client.BaseRunnerRequestException - ) or isinstance(exception, RunnerFailedToStart): + exception, jsonrpc_client.BaseRunnerRequestException + ) or isinstance(exception, jsonrpc_client.RunnerFailedToStart): logger.error(exception.message) else: logger.error("Unexpected exception:") logger.exception(exception) - raise RunnerFailedToStart( + raise jsonrpc_client.RunnerFailedToStart( "Failed to initialize runner(s). See previous logs for more details" ) for project in projects: - if project_status != domain.ProjectStatus.CONFIG_VALID: + if project.status != domain.ProjectStatus.CONFIG_VALID: continue try: @@ -251,18 +274,20 @@ async def start_runners_with_presets( project_path=project.dir_path, ws_context=ws_context ) except config_models.ConfigurationError as exception: - raise RunnerFailedToStart( + raise jsonrpc_client.RunnerFailedToStart( f"Reading project config with presets and collecting actions in {project.dir_path} failed: {exception.message}" ) - + # update config of dev_workspace runner, the new config contains resolved presets - dev_workspace_runner = ws_context.ws_projects_extension_runners[project.dir_path]['dev_workspace'] + dev_workspace_runner = ws_context.ws_projects_extension_runners[ + project.dir_path + ]["dev_workspace"] await update_runner_config(runner=dev_workspace_runner, project=project) async def get_or_start_runners_with_presets( project_dir_path: Path, ws_context: context.WorkspaceContext -) -> runner_info.ExtensionRunnerInfo: +) -> runner_client.ExtensionRunnerInfo: # project is expected to have status `ProjectStatus.CONFIG_VALID` has_dev_workspace_runner = ( "dev_workspace" in ws_context.ws_projects_extension_runners[project_dir_path] @@ -273,36 +298,35 @@ async def get_or_start_runners_with_presets( dev_workspace_runner = ws_context.ws_projects_extension_runners[project_dir_path][ "dev_workspace" ] - if dev_workspace_runner.status == runner_info.RunnerStatus.RUNNING: + if dev_workspace_runner.status == runner_client.RunnerStatus.RUNNING: return dev_workspace_runner - elif dev_workspace_runner.status == runner_info.RunnerStatus.INITIALIZING: + elif dev_workspace_runner.status == runner_client.RunnerStatus.INITIALIZING: await dev_workspace_runner.initialized_event.wait() return dev_workspace_runner else: - raise RunnerFailedToStart( + raise jsonrpc_client.RunnerFailedToStart( f"Status of dev_workspace runner: {dev_workspace_runner.status}, logs: {dev_workspace_runner.logs_path}" ) async def start_runner( project_def: domain.Project, env_name: str, ws_context: context.WorkspaceContext -) -> runner_info.ExtensionRunnerInfo: +) -> runner_client.ExtensionRunnerInfo: # this function manages status of the runner and initialized event - runner = await _start_extension_runner_process( - runner_dir=project_def.dir_path, env_name=env_name, ws_context=ws_context + runner = runner_client.ExtensionRunnerInfo( + working_dir_path=project_def.dir_path, + env_name=env_name, + status=runner_client.RunnerStatus.INITIALIZING, + initialized_event=asyncio.Event(), + client=None, ) - - if runner is None: - raise RunnerFailedToStart( - f"Runner '{env_name}' in project {project_def.name} failed to start" - ) - - runner.status = runner_info.RunnerStatus.INITIALIZING save_runner_in_context(runner=runner, ws_context=ws_context) + await _start_extension_runner_process(runner=runner, ws_context=ws_context) + try: await _init_lsp_client(runner=runner, project=project_def) - except RunnerFailedToStart as exception: - runner.status = runner_info.RunnerStatus.FAILED + except jsonrpc_client.RunnerFailedToStart as exception: + runner.status = runner_client.RunnerStatus.FAILED await notify_project_changed(project_def) runner.initialized_event.set() raise exception @@ -319,17 +343,17 @@ async def start_runner( project_path=project_def.dir_path, ws_context=ws_context ) except config_models.ConfigurationError as exception: - runner.status = runner_info.RunnerStatus.FAILED + runner.status = runner_client.RunnerStatus.FAILED runner.initialized_event.set() await notify_project_changed(project_def) - raise RunnerFailedToStart( + raise jsonrpc_client.RunnerFailedToStart( f"Found problem in configuration of {project_def.dir_path}: {exception.message}" ) await update_runner_config(runner=runner, project=project_def) await _finish_runner_init(runner=runner, project=project_def, ws_context=ws_context) - runner.status = runner_info.RunnerStatus.RUNNING + runner.status = runner_client.RunnerStatus.RUNNING await notify_project_changed(project_def) runner.initialized_event.set() @@ -338,27 +362,32 @@ async def start_runner( async def get_or_start_runner( project_def: domain.Project, env_name: str, ws_context: context.WorkspaceContext -) -> runner_info.ExtensionRunnerInfo: +) -> runner_client.ExtensionRunnerInfo: runners_by_env = ws_context.ws_projects_extension_runners[project_def.dir_path] try: runner = runners_by_env[env_name] + logger.trace(f"Runner {runner.readable_id} found") except KeyError: + logger.trace( + f"Runner for env {env_name} in {project_def.dir_path} not found, start one" + ) runner = await start_runner( project_def=project_def, env_name=env_name, ws_context=ws_context ) - if runner.status != runner_info.RunnerStatus.RUNNING: + if runner.status != runner_client.RunnerStatus.RUNNING: runner_error = False - if runner.status == runner_info.RunnerStatus.INITIALIZING: + if runner.status == runner_client.RunnerStatus.INITIALIZING: + logger.trace(f"Runner {runner.readable_id} is initializing, wait for it") await runner.initialized_event.wait() - if runner.status != runner_info.RunnerStatus.RUNNING: + if runner.status != runner_client.RunnerStatus.RUNNING: runner_error = True else: runner_error = True if runner_error: - raise RunnerFailedToStart( + raise jsonrpc_client.RunnerFailedToStart( f"Runner {env_name} in project {project_def.dir_path} is not running. Status: {runner.status}" ) @@ -367,31 +396,33 @@ async def get_or_start_runner( async def _start_dev_workspace_runner( project_def: domain.Project, ws_context: context.WorkspaceContext -) -> runner_info.ExtensionRunnerInfo: +) -> runner_client.ExtensionRunnerInfo: return await start_runner( project_def=project_def, env_name="dev_workspace", ws_context=ws_context ) async def _init_lsp_client( - runner: runner_info.ExtensionRunnerInfo, project: domain.Project + runner: runner_client.ExtensionRunnerInfo, project: domain.Project ) -> None: try: - await runner_client.initialize( - runner, + await _internal_client_api.initialize( + client=runner.client, client_process_id=os.getpid(), client_name="FineCode_WorkspaceManager", client_version="0.1.0", ) - except runner_client.BaseRunnerRequestException as error: - raise RunnerFailedToStart(f"Runner failed to initialize: {error.message}") + except jsonrpc_client.BaseRunnerRequestException as error: + raise jsonrpc_client.RunnerFailedToStart( + f"Runner failed to initialize: {error.message}" + ) try: - await runner_client.notify_initialized(runner) + await _internal_client_api.notify_initialized(runner.client) except Exception as error: logger.error(f"Failed to notify runner about initialization: {error}") logger.exception(error) - raise RunnerFailedToStart( + raise jsonrpc_client.RunnerFailedToStart( f"Runner failed to notify about initialization: {error}" ) @@ -399,7 +430,7 @@ async def _init_lsp_client( async def update_runner_config( - runner: runner_info.ExtensionRunnerInfo, project: domain.Project + runner: runner_client.ExtensionRunnerInfo, project: domain.Project ) -> None: assert project.actions is not None config = runner_client.RunnerConfig( @@ -407,21 +438,19 @@ async def update_runner_config( ) try: await runner_client.update_config(runner, project.def_path, config) - except runner_client.BaseRunnerRequestException as exception: - runner.status = runner_info.RunnerStatus.FAILED + except jsonrpc_client.BaseRunnerRequestException as exception: + runner.status = runner_client.RunnerStatus.FAILED await notify_project_changed(project) runner.initialized_event.set() - raise RunnerFailedToStart( + raise jsonrpc_client.RunnerFailedToStart( f"Runner failed to update config: {exception.message}" ) - logger.debug( - f"Updated config of runner {runner.readable_id}, process id {runner.process_id}" - ) + logger.debug(f"Updated config of runner {runner.readable_id}") async def _finish_runner_init( - runner: runner_info.ExtensionRunnerInfo, + runner: runner_client.ExtensionRunnerInfo, project: domain.Project, ws_context: context.WorkspaceContext, ) -> None: @@ -433,7 +462,7 @@ async def _finish_runner_init( def save_runners_from_tasks_in_context( tasks: list[asyncio.Task], ws_context: context.WorkspaceContext ) -> None: - extension_runners: list[runner_info.ExtensionRunnerInfo] = [ + extension_runners: list[runner_client.ExtensionRunnerInfo] = [ runner.result() for runner in tasks if runner is not None ] @@ -442,7 +471,7 @@ def save_runners_from_tasks_in_context( def save_runner_in_context( - runner: runner_info.ExtensionRunnerInfo, ws_context: context.WorkspaceContext + runner: runner_client.ExtensionRunnerInfo, ws_context: context.WorkspaceContext ) -> None: if runner.working_dir_path not in ws_context.ws_projects_extension_runners: ws_context.ws_projects_extension_runners[runner.working_dir_path] = {} @@ -452,7 +481,8 @@ def save_runner_in_context( async def send_opened_files( - runner: runner_info.ExtensionRunnerInfo, opened_files: list[domain.TextDocumentInfo] + runner: runner_client.ExtensionRunnerInfo, + opened_files: list[domain.TextDocumentInfo], ): files_for_runner: list[domain.TextDocumentInfo] = [] for opened_file_info in opened_files: @@ -499,7 +529,7 @@ async def check_runner(runner_dir: Path, env_name: str) -> bool: raw_stdout, raw_stderr = await asyncio.wait_for( async_subprocess.communicate(), timeout=5 ) - except asyncio.TimeoutError: + except TimeoutError: logger.debug(f"Timeout 5 sec({runner_dir})") return False diff --git a/src/finecode/services/run_service/__init__.py b/src/finecode/services/run_service/__init__.py index a31ae23..f6e94f0 100644 --- a/src/finecode/services/run_service/__init__.py +++ b/src/finecode/services/run_service/__init__.py @@ -1,4 +1,4 @@ -from .exceptions import ActionRunFailed +from .exceptions import ActionRunFailed, StartingEnvironmentsFailed from .proxy_utils import ( run_action, find_action_project_and_run, diff --git a/src/finecode/services/run_service/proxy_utils.py b/src/finecode/services/run_service/proxy_utils.py index a208bf4..27786bd 100644 --- a/src/finecode/services/run_service/proxy_utils.py +++ b/src/finecode/services/run_service/proxy_utils.py @@ -4,15 +4,15 @@ import collections.abc import contextlib import pathlib -from typing import Any +import typing import ordered_set from loguru import logger from finecode import context, domain, find_project, user_messages -from finecode.runner import manager as runner_manager -from finecode.runner import runner_client, runner_info -from finecode.runner.manager import RunnerFailedToStart +from finecode.runner import runner_manager +from finecode.runner import runner_client +from finecode.runner.runner_manager import RunnerFailedToStart from finecode.runner.runner_client import RunResultFormat # reexport from finecode.services.run_service import payload_preprocessor @@ -34,7 +34,7 @@ async def find_action_project( raise error except ValueError as error: logger.warning(f"Skip {action_name} on {file_path}: {error}") - raise ActionRunFailed(error) + raise ActionRunFailed(error) from error project_status = ws_context.ws_projects[project_path].status if project_status != domain.ProjectStatus.CONFIG_VALID: @@ -53,7 +53,7 @@ async def find_action_project( async def find_action_project_and_run( file_path: pathlib.Path, action_name: str, - params: dict[str, Any], + params: dict[str, typing.Any], ws_context: context.WorkspaceContext, ) -> runner_client.RunActionResponse: project_path = await find_action_project( @@ -77,9 +77,9 @@ async def find_action_project_and_run( async def run_action_in_runner( action_name: str, - params: dict[str, Any], - runner: runner_info.ExtensionRunnerInfo, - options: dict[str, Any] | None = None, + params: dict[str, typing.Any], + runner: runner_client.ExtensionRunnerInfo, + options: dict[str, typing.Any] | None = None, ) -> runner_client.RunActionResponse: try: response = await runner_client.run_action( @@ -137,9 +137,9 @@ async def __anext__(self) -> T: async def run_action_and_notify( action_name: str, - params: dict[str, Any], + params: dict[str, typing.Any], partial_result_token: int | str, - runner: runner_info.ExtensionRunnerInfo, + runner: runner_client.ExtensionRunnerInfo, result_list: AsyncList, partial_results_task: asyncio.Task, ) -> runner_client.RunActionResponse: @@ -170,7 +170,7 @@ async def get_partial_results( @contextlib.asynccontextmanager async def run_with_partial_results( action_name: str, - params: dict[str, Any], + params: dict[str, typing.Any], partial_result_token: int | str, project_dir_path: pathlib.Path, ws_context: context.WorkspaceContext, @@ -188,7 +188,9 @@ async def run_with_partial_results( result_list=result, partial_result_token=partial_result_token ) ) - action = next(action for action in project.actions if action.name == action_name) + action = next( + action for action in project.actions if action.name == action_name + ) action_envs = ordered_set.OrderedSet( [handler.env for handler in action.handlers] ) @@ -200,7 +202,7 @@ async def run_with_partial_results( except runner_manager.RunnerFailedToStart as exception: raise ActionRunFailed( f"Runner {env_name} in project {project.dir_path} failed: {exception.message}" - ) + ) from exception tg.create_task( run_action_and_notify( @@ -233,7 +235,7 @@ async def run_with_partial_results( async def find_action_project_and_run_with_partial_results( file_path: pathlib.Path, action_name: str, - params: dict[str, Any], + params: dict[str, typing.Any], partial_result_token: int | str, ws_context: context.WorkspaceContext, ) -> collections.abc.AsyncIterator[runner_client.RunActionRawResult]: @@ -302,47 +304,74 @@ async def start_required_environments( project_required_envs.add(handler.env) required_envs_by_project[project_dir_path] = project_required_envs - # start runners for required environments that aren't already running - for project_dir_path, required_envs in required_envs_by_project.items(): - project = ws_context.ws_projects[project_dir_path] - existing_runners = ws_context.ws_projects_extension_runners.get( - project_dir_path, {} - ) - - for env_name in required_envs: - runner_exist = env_name in existing_runners - start_runner = True - if runner_exist: - runner_is_running = ( - existing_runners[env_name].status - == runner_info.RunnerStatus.RUNNING + try: + async with asyncio.TaskGroup() as tg: + # start runners for required environments that aren't already running + for project_dir_path, required_envs in required_envs_by_project.items(): + project = ws_context.ws_projects[project_dir_path] + existing_runners = ws_context.ws_projects_extension_runners.get( + project_dir_path, {} ) - start_runner = not runner_is_running - if start_runner: - try: - runner = await runner_manager.start_runner( - project_def=project, env_name=env_name, ws_context=ws_context - ) - except runner_manager.RunnerFailedToStart as e: - raise StartingEnvironmentsFailed( - f"Failed to start runner for env '{env_name}' in project '{project.name}': {e}" + for env_name in required_envs: + tg.create_task( + _start_runner_or_update_config( + env_name=env_name, + existing_runners=existing_runners, + project=project, + update_config_in_running_runners=update_config_in_running_runners, + ws_context=ws_context, + ) ) + except ExceptionGroup as eg: + errors: list[str] = [] + for exception in eg.exceptions: + if isinstance(exception, StartingEnvironmentsFailed): + errors.append(exception.message) else: - if update_config_in_running_runners: - runner = existing_runners[env_name] - logger.trace( - f"Runner {runner.readable_id} is running already, update config" - ) + errors.append(str(exception)) + raise StartingEnvironmentsFailed(".".join(errors)) - try: - await runner_manager.update_runner_config( - runner=runner, project=project - ) - except RunnerFailedToStart as exception: - raise StartingEnvironmentsFailed( - f"Failed to update config of runner {runner.readable_id}" - ) + +async def _start_runner_or_update_config( + env_name: str, + existing_runners: dict[str, runner_client.ExtensionRunnerInfo], + project: domain.Project, + update_config_in_running_runners: bool, + ws_context: context.WorkspaceContext, +): + runner_exist = env_name in existing_runners + start_runner = True + if runner_exist: + runner_is_running = ( + existing_runners[env_name].status == runner_client.RunnerStatus.RUNNING + ) + start_runner = not runner_is_running + + if start_runner: + try: + await runner_manager.start_runner( + project_def=project, env_name=env_name, ws_context=ws_context + ) + except runner_manager.RunnerFailedToStart as exception: + raise StartingEnvironmentsFailed( + f"Failed to start runner for env '{env_name}' in project '{project.name}': {exception.message}" + ) from exception + else: + if update_config_in_running_runners: + runner = existing_runners[env_name] + logger.trace( + f"Runner {runner.readable_id} is running already, update config" + ) + + try: + await runner_manager.update_runner_config( + runner=runner, project=project + ) + except RunnerFailedToStart as exception: + raise StartingEnvironmentsFailed( + f"Failed to update config of runner {runner.readable_id}" + ) from exception async def run_actions_in_running_project( @@ -575,7 +604,7 @@ async def _run_action_in_env_runner( ) raise ActionRunFailed( f"Action {action_name} failed in {runner.readable_id}: {error.message} . Log file: {runner.logs_path}" - ) + ) from error return response diff --git a/src/finecode/services/shutdown_service.py b/src/finecode/services/shutdown_service.py index eda2fa5..8b2d2db 100644 --- a/src/finecode/services/shutdown_service.py +++ b/src/finecode/services/shutdown_service.py @@ -1,15 +1,14 @@ from loguru import logger from finecode import context -from finecode.runner import manager as runner_manager -from finecode.runner import runner_info +from finecode.runner import runner_client, runner_manager def on_shutdown(ws_context: context.WorkspaceContext): running_runners = [] for runners_by_env in ws_context.ws_projects_extension_runners.values(): for runner in runners_by_env.values(): - if runner.status == runner_info.RunnerStatus.RUNNING: + if runner.status == runner_client.RunnerStatus.RUNNING: running_runners.append(runner) logger.trace(f"Stop all {len(running_runners)} running extension runners") @@ -17,4 +16,8 @@ def on_shutdown(ws_context: context.WorkspaceContext): for runner in running_runners: runner_manager.stop_extension_runner_sync(runner=runner) + if ws_context.runner_io_thread is not None: + logger.trace("Stop IO thread") + ws_context.runner_io_thread.stop(timeout=5) + # TODO: stop MCP if running diff --git a/src/finecode/utils/async_proc_queue.py b/src/finecode/utils/async_proc_queue.py deleted file mode 100644 index ad77ed7..0000000 --- a/src/finecode/utils/async_proc_queue.py +++ /dev/null @@ -1,73 +0,0 @@ -import asyncio -import queue -from concurrent.futures import ThreadPoolExecutor -from multiprocessing import Manager, cpu_count -from typing import Generic, TypeVar, cast - - -class QueueEnd: - # just object() would not support multiprocessing, use class and compare by it - def __eq__(self, other): - return self.__class__ == other.__class__ - - -QueueItemType = TypeVar("QueueItemType") - - -class _ProcQueue(Generic[QueueItemType]): - def __init__(self, q: queue.Queue[QueueItemType]): - self._queue: queue.Queue[QueueItemType] = q - self._real_executor = None - self._cancelled_join = False - - @property - def _executor(self): - if not self._real_executor: - self._real_executor = ThreadPoolExecutor(max_workers=cpu_count()) - return self._real_executor - - def __getstate__(self): - self_dict = self.__dict__ - self_dict["_real_executor"] = None - return self_dict - - def qsize(self) -> int: - return self._queue.qsize() - - def empty(self) -> bool: - return self._queue.empty() - - def full(self) -> bool: - return self._queue.full() - - def put(self, item: QueueItemType) -> None: - self._queue.put(item) - - def put_nowait(self, item: QueueItemType) -> None: - self._queue.put_nowait(item) - - def get(self) -> QueueItemType: - return self._queue.get() - - def get_nowait(self) -> QueueItemType: - return self._queue.get_nowait() - - def task_done(self) -> None: - self._queue.task_done() - - async def put_async(self, item: QueueItemType): - loop = asyncio.get_event_loop() - return await loop.run_in_executor(self._executor, self.put, item) - - async def get_async(self) -> QueueItemType: - loop = asyncio.get_event_loop() - return await loop.run_in_executor(self._executor, self.get) - - -AsyncQueue = _ProcQueue - - -def create_async_process_queue(maxsize=0) -> AsyncQueue: - m = Manager() - q = m.Queue(maxsize=maxsize) - return cast(AsyncQueue, _ProcQueue(q))