Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion servers/evaluation/src/evaluation.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,14 @@
from datetime import datetime
from typing import Any, Callable, Dict, List, Optional

from rouge_score import rouge_scorer
try:
from rouge_score import rouge_scorer
except ModuleNotFoundError as exc:
raise ModuleNotFoundError(
"Missing optional dependency 'rouge-score' for the evaluation server. "
"Install it with `uv sync --extra evaluation` or "
"`pip install \"ultrarag[evaluation]\"`."
) from exc
from tabulate import tabulate

from ultrarag.server import UltraRAG_MCP_Server
Expand Down
38 changes: 32 additions & 6 deletions src/ultrarag/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import yaml
from dotenv import load_dotenv
from fastmcp import Client
from fastmcp.exceptions import ToolError

from ultrarag.cli import log_server_banner
from ultrarag.mcp_exceptions import (
Expand All @@ -26,6 +27,12 @@
logger = None
PipelineStep = Union[str, Dict[str, Any]]
node_status = False
SERVER_OPTIONAL_EXTRAS = {
"retriever": "retriever",
"generation": "generation",
"evaluation": "evaluation",
"corpus": "corpus",
}


class MockContent:
Expand Down Expand Up @@ -955,6 +962,29 @@ async def build(config_path: str) -> None:

async def build_steps(steps: List[PipelineStep]):
nonlocal already_built, parameter_all, server_all

async def call_build_tool(srv_name: str, full_tool: str) -> None:
try:
await client.call_tool(
full_tool, {"parameter_file": parameter_path[srv_name]}
)
except ToolError as exc:
extra_name = SERVER_OPTIONAL_EXTRAS.get(srv_name)
msg = str(exc)
if (
"Server session was closed unexpectedly" in msg
and extra_name is not None
):
raise RuntimeError(
"[UltraRAG Error] Failed to start server "
f"'{srv_name}' while building pipeline. "
"This usually means optional dependencies are missing. "
"Install them and retry: "
f"`uv sync --extra {extra_name}` "
f"(or `pip install \"ultrarag[{extra_name}]\"`)."
) from exc
raise

for step in steps:
if isinstance(step, str):
srv_name, tool_name = step.split(".")
Expand All @@ -965,9 +995,7 @@ async def build_steps(steps: List[PipelineStep]):
server_all[srv_name] = {
"prompts" if srv_name == "prompt" else "tools": {}
}
await client.call_tool(
full_tool, {"parameter_file": parameter_path[srv_name]}
)
await call_build_tool(srv_name, full_tool)
logger.info(f"server.yaml for {srv_name} has been built already")
param = loader.load_parameter_config(parameter_path[srv_name])
serv = loader.load_parameter_config(
Expand Down Expand Up @@ -1018,9 +1046,7 @@ async def build_steps(steps: List[PipelineStep]):
server_all[srv_name] = {
"prompts" if srv_name == "prompt" else "tools": {}
}
await client.call_tool(
full_tool, {"parameter_file": parameter_path[srv_name]}
)
await call_build_tool(srv_name, full_tool)
logger.info(
f"server.yaml for {srv_name} has been built already"
)
Expand Down