diff --git a/.gitignore b/.gitignore index c6b64d1dd..f9cc46524 100644 --- a/.gitignore +++ b/.gitignore @@ -169,3 +169,4 @@ ms_agent/app/temp_workspace/ webui/work_dir/ .ms_agent_snapshots/ +.ms_agent/ diff --git a/ms_agent/agent/base.py b/ms_agent/agent/base.py index 0c6ee3fc3..090b7ddde 100644 --- a/ms_agent/agent/base.py +++ b/ms_agent/agent/base.py @@ -51,6 +51,19 @@ def __init__(self, self.config.output_dir = self.output_dir except Exception: pass + # Merge the work-dir project patch (e.g. a persisted /model override) so + # config overrides round-trip from /.ms_agent/config.yaml — + # anchored to the project (the work dir), not the config file's + # directory. This keeps running a shared/template config from picking up + # (or scattering) overrides in that config's folder. + try: + from omegaconf import OmegaConf + from ms_agent.config.resolver import ConfigResolver + patch = ConfigResolver()._load_project_patch(self.output_dir) + if patch is not None: + self.config = OmegaConf.merge(self.config, patch) + except Exception: + pass @abstractmethod async def run( diff --git a/ms_agent/agent/llm_agent.py b/ms_agent/agent/llm_agent.py index 44e34e250..c5448c1ea 100644 --- a/ms_agent/agent/llm_agent.py +++ b/ms_agent/agent/llm_agent.py @@ -38,6 +38,11 @@ from ms_agent.skill.skill_tools import SkillToolSet from ms_agent.utils.snapshot import take_snapshot from ms_agent.utils.task_manager import TaskManager +from ms_agent.ui.events import (ContentDelta, ContentEnd, ContextCompacted, + ErrorRaised, PlanEntry, PlanUpdated, + ReasoningDelta, ReasoningEnded, ReasoningStarted, + ToolCallCompleted, ToolCallStarted, + TurnCompleted, UsageInfo) from ..config.config import Config, ConfigLifecycleHandler from .base import Agent @@ -172,6 +177,24 @@ def __init__( # Gates both the initial >>> prompt and the mid-turn InputCallback. self._interactive = False + # Optional injected PermissionHandler (TUI/WebUI/Server). When None, + # _select_permission_handler() picks by mode + interactivity. + self._permission_handler = kwargs.get('permission_handler', None) + + # Structured event sink — the UI-agnostic output seam (ms_agent.ui). + # When set, the agent emits semantic AgentEvents (content / reasoning / + # tool / ...) that a TUI renders and a WebUI backend forwards. When + # None, output goes to stdout (current CLI behavior, byte-identical). + # This is the seam that validates the WebUI contract. + self._event_sink = kwargs.get('event_sink', None) + + # Async input source (awaitable read_prompt). When set, the interactive + # loop reads the next prompt through it (TUI now, WebUI queue later), + # so the read never blocks the event loop or fights the renderer for + # the terminal — the enabling seam for the native (route-A) lifecycle. + # When None, the legacy sync console_io / input() path is used. + self._input_source = kwargs.get('input_source', None) + # Personalization (lazy-loaded in _build_personalization_section) self._profile_manager = ProfileManager() @@ -238,6 +261,18 @@ async def prepare_skills(self): self._plugin_runtime.skill_runtime = self._skill_runtime self._plugin_runtime._sync_skill_runtime(self.config) + # Wire /skill-name slash commands. Without this the SkillCommandBridge + # is never registered and `/skill-id args` falls through to the model as + # raw text (only "works" when the skill happens to be in the system + # prompt). Registering it makes `/skill-id` dispatch to SUBMIT_PROMPT, + # including *disabled* skills (per meeting decision). The bridge is the + # only router interceptor (plugins use register()), so reset first to + # rebind to the current catalog across restarts without duplicates. + from ms_agent.command.skill_bridge import SkillCommandBridge + router = self._get_command_router() + router._interceptors.clear() + SkillCommandBridge(self._skill_catalog).register(router) + def _build_system_content(self) -> str: """Build the full system prompt content. @@ -456,7 +491,9 @@ def register_callback_from_config(self): if self._interactive: self.callbacks.append(callbacks_mapping[_callback]( self.config, - command_router=self._get_command_router())) + command_router=self._get_command_router(), + input_source=self._input_source, + event_sink=self._event_sink)) else: self.callbacks.append(callbacks_mapping[_callback]( self.config)) @@ -469,7 +506,9 @@ def register_callback_from_config(self): self.callbacks.append( input_cls( self.config, - command_router=self._get_command_router())) + command_router=self._get_command_router(), + input_source=self._input_source, + event_sink=self._event_sink)) async def on_task_begin(self, messages: List[Message]): self.log_output(f'Agent {self.tag} task beginning.') @@ -573,16 +612,39 @@ async def parallel_tool_call(self, self.log_output(_new_message.content) return messages + def _select_permission_handler(self, mode: str): + """Pick the PermissionHandler by mode + runtime environment. + + - An explicitly injected handler (``set_permission_handler`` / the + ``permission_handler`` kwarg) always wins — this is how the TUI / + WebUI / Server supply their own confirmation UI. + - ``interactive`` (alias ``restricted``) in an interactive terminal + session -> ``CLIPermissionHandler`` (the ``[y/s/a/e/n]`` prompt), + so non-whitelisted tools actually ask the user. + - Everything else (``auto`` / ``strict`` / non-interactive) -> + ``AutoPermissionHandler`` (SafetyGuard still enforces the floor). + """ + if self._permission_handler is not None: + return self._permission_handler + from ms_agent.permission import ( + AutoPermissionHandler, + CLIPermissionHandler, + ) + # ``PermissionConfig.from_dict`` already normalizes ``restricted`` -> + # ``interactive``; accept both so a direct caller passing the raw alias + # still gets the interactive prompt (not a silent AutoPermissionHandler). + if mode in ('interactive', 'restricted') and self._interactive: + return CLIPermissionHandler() + return AutoPermissionHandler() + def _build_permission_objects(self): """Create SafetyGuard and PermissionEnforcer from config if configured.""" from ms_agent.permission import ( - AutoPermissionHandler, PermissionConfig, PermissionEnforcer, PermissionMemory, SafetyGuard, ) - from ms_agent.permission.config import SafetyConfig raw = {} if hasattr(self.config, 'permission'): @@ -605,12 +667,41 @@ def _build_permission_objects(self): workspace_root=workspace_root, ) - handler = AutoPermissionHandler() + handler = self._select_permission_handler(perm_config.mode) memory = PermissionMemory(project_path=workspace_root) enforcer = PermissionEnforcer(config=perm_config, handler=handler, memory=memory) return safety_guard, enforcer, perm_config + def set_permission_handler(self, handler) -> None: + """Inject a custom PermissionHandler (TUI/WebUI/Server) before run.""" + self._permission_handler = handler + + def set_permission_mode(self, mode: str) -> str: + """Change the permission mode at runtime; returns the normalized mode. + + Mutates the live ToolManager + enforcer so the next tool call uses the + new mode without rebuilding the agent (``PermissionConfig`` is frozen, + so a replaced copy is swapped in). ``restricted`` normalizes to + ``interactive`` (the canonical asking mode). + """ + from dataclasses import replace + mode = {'restricted': 'interactive'}.get(mode, mode) + if mode not in ('auto', 'strict', 'interactive'): + raise ValueError( + f"Unknown permission mode '{mode}' " + '(auto | restricted | strict | interactive)') + tm = self.tool_manager + if tm is not None: + tm._permission_mode = mode + if getattr(tm, '_permission_config', None) is not None: + tm._permission_config = replace( + tm._permission_config, mode=mode) + enf = getattr(tm, '_permission_enforcer', None) + if enf is not None and getattr(enf, '_config', None) is not None: + enf._config = replace(enf._config, mode=mode) + return mode + async def prepare_tools(self): """Initialize and connect the tool manager.""" import uuid @@ -788,6 +879,72 @@ def _write_thinking_footer(self): stream.write(f'\n{self._THINKING_SEP}\n') stream.flush() + # ── output seam ──────────────────────────────────────────────────────── + # These route each output write to the structured event sink when one is + # injected, else to stdout. The no-sink branch calls the exact same code as + # before the seam existed, so CLI output stays byte-identical. + + def _emit_reasoning_start(self) -> None: + if self._event_sink is not None: + self._event_sink.emit(ReasoningStarted()) + else: + self._write_thinking_header() + + def _emit_reasoning_delta(self, text: str) -> None: + if self._event_sink is not None: + self._event_sink.emit(ReasoningDelta(text)) + else: + self._write_reasoning(text, dim=True) + + def _emit_reasoning_end(self) -> None: + if self._event_sink is not None: + self._event_sink.emit(ReasoningEnded()) + else: + self._write_thinking_footer() + + def _emit_content(self, text: str) -> None: + if self._event_sink is not None: + self._event_sink.emit(ContentDelta(text)) + else: + sys.stdout.write(text) + sys.stdout.flush() + + def _emit_content_end(self) -> None: + if self._event_sink is not None: + self._event_sink.emit(ContentEnd()) + else: + sys.stdout.write('\n') + + @staticmethod + def _extract_plan_from_tool_result(msg): + """Parse a todo / split_task tool result into a list of PlanEntry, or + None. Feeds the WebUI plan/todo panel (and the TUI plan render). Mirrors + the ACP server's plan extraction so both consumers agree.""" + name = getattr(msg, 'name', '') or '' + short = name.split('---')[-1] if '---' in name else name + if 'todo' not in short and short != 'split_task': + return None + content = msg.content if isinstance(msg.content, str) else str( + msg.content) + try: + data = json.loads(content) + except (json.JSONDecodeError, TypeError, ValueError): + return None + todos = (data.get('todos') if isinstance(data, dict) + else data if isinstance(data, list) else None) + if not isinstance(todos, list): + return None + entries = [] + for it in todos: + if isinstance(it, dict): + entries.append( + PlanEntry( + content=str( + it.get('content') or it.get('description') + or it.get('task') or ''), + status=str(it.get('status') or 'pending'))) + return entries or None + @property def system(self): return getattr( @@ -843,6 +1000,24 @@ def _resolve_interactive(self, messages) -> bool: return False return sys.stdin.isatty() + def _has_restorable_history(self) -> bool: + """True when this run should resume from the session log rather than + read an initial prompt. + + Guards against the prompt-then-discard on resume: without this, the + interactive first ``>>>`` read happens *before* history restore, so the + user's first line would be thrown away when the log is loaded. When a + resumable log exists (``load_cache`` + non-empty ``SessionLog``), skip + the initial prompt; restore seeds context and the loop then prompts for + the next turn via ``InputCallback``. + """ + if not self.load_cache or self.session_log is None: + return False + try: + return bool(self.session_log.get_all_messages()) + except Exception: + return False + async def create_messages( self, messages: Union[List[Message], str]) -> List[Message]: """ @@ -1053,10 +1228,20 @@ def _init_session_log(self) -> None: session_cfg, 'dir', None ) if session_cfg else None if session_dir is None: - session_dir = os.path.join( - getattr(self.config, 'output_dir', 'output'), - 'sessions', - ) + # Sessions live globally, keyed by the work dir (Claude Code style), + # decoupled from output_dir. An explicit session_log.dir (set by the + # WebUI/Server per project) still wins. Legacy /sessions + # is migrated forward if present. + from ms_agent.project.paths import global_sessions_dir + output_dir = getattr(self.config, 'output_dir', 'output') + session_dir = str(global_sessions_dir(output_dir)) + legacy_dir = os.path.join(output_dir, 'sessions') + if os.path.isdir(legacy_dir) and not os.path.isdir(session_dir): + try: + import shutil + shutil.copytree(legacy_dir, session_dir) + except Exception: + pass session_key = getattr(session_cfg, 'session_key', None) if session_cfg else None self.session_log = SessionLog(session_dir, session_key=session_key) @@ -1299,35 +1484,34 @@ async def step( new_reasoning = reasoning_text[len(_reasoning):] if new_reasoning: if not _printed_reasoning_header: - self._write_thinking_header() + self._emit_reasoning_start() _printed_reasoning_header = True - self._write_reasoning(new_reasoning, dim=True) + self._emit_reasoning_delta(new_reasoning) _reasoning = reasoning_text new_content = _response_message.content[len(_content):] if self.stream_output and new_content: if _printed_reasoning_header and not _printed_reasoning_footer: - self._write_thinking_footer() + self._emit_reasoning_end() _printed_reasoning_footer = True - sys.stdout.write(new_content) - sys.stdout.flush() + self._emit_content(new_content) _content = _response_message.content messages[-1] = _response_message yield messages if self.stream_output: if _printed_reasoning_header and not _printed_reasoning_footer: - self._write_thinking_footer() + self._emit_reasoning_end() # Handle reasoning summaries that arrive after content if self.show_reasoning and _response_message is not None: final_reasoning = getattr(_response_message, 'reasoning_content', '') or '' if final_reasoning and not _printed_reasoning_header: - self._write_thinking_header() - self._write_reasoning(final_reasoning, dim=True) - self._write_thinking_footer() + self._emit_reasoning_start() + self._emit_reasoning_delta(final_reasoning) + self._emit_reasoning_end() - sys.stdout.write('\n') + self._emit_content_end() else: _response_message = self.llm.generate(messages, tools=tools) if self.show_reasoning: @@ -1335,10 +1519,13 @@ async def step( getattr(_response_message, 'reasoning_content', '') or '') if reasoning_text: - self._write_thinking_header() - self._write_reasoning(reasoning_text, dim=True) - self._write_thinking_footer() + self._emit_reasoning_start() + self._emit_reasoning_delta(reasoning_text) + self._emit_reasoning_end() if _response_message.content: + if self._event_sink is not None: + self._emit_content(_response_message.content) + self._emit_content_end() self.log_output('[assistant]:') self.log_output(_response_message.content) @@ -1353,7 +1540,31 @@ async def step( self.save_history(messages) if _response_message.tool_calls: + if self._event_sink is not None: + for tc in _response_message.tool_calls: + self._event_sink.emit( + ToolCallStarted( + call_id=str(tc.get('id') or ''), + name=str( + tc.get('tool_name') or tc.get('name') or ''), + arguments=tc.get('arguments'))) + _tool_start = len(messages) messages = await self.parallel_tool_call(messages) + if self._event_sink is not None: + for m in messages[_tool_start:]: + if getattr(m, 'role', None) == 'tool': + _content = (m.content if isinstance(m.content, str) + else str(m.content)) + self._event_sink.emit( + ToolCallCompleted( + call_id=str( + getattr(m, 'tool_call_id', '') or ''), + name=str(getattr(m, 'name', '') or ''), + result=_content or '')) + # todo/split_task tool results drive the plan panel. + _plan = self._extract_plan_from_tool_result(m) + if _plan is not None: + self._event_sink.emit(PlanUpdated(entries=_plan)) # usage # NOTE: token accounting must run BEFORE after_tool_call. The interactive @@ -1401,6 +1612,15 @@ async def step( f'total_cache_created: {LLMAgent.TOTAL_CACHE_CREATION_INPUT_TOKENS}' ) + if self._event_sink is not None: + self._event_sink.emit( + TurnCompleted(usage=UsageInfo( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + reasoning_tokens=reasoning_tokens, + total_prompt_tokens=LLMAgent.TOTAL_PROMPT_TOKENS, + total_completion_tokens=LLMAgent.TOTAL_COMPLETION_TOKENS))) + yield messages def prepare_llm(self): @@ -1602,16 +1822,27 @@ async def run_loop(self, messages: Union[List[Message], str], if configured: messages = configured elif self._interactive: - from ms_agent.command.interactive import InteractiveSession - session = InteractiveSession(self._get_command_router()) - turn = await session.run_turn( - messages=None, runtime=self.runtime) - if turn.action == 'quit': - # Exited at the interactive prompt without a task. - self.runtime.should_stop = True - await self.cleanup_tools() - return - messages = turn.text + # On resume (restorable history), skip the initial prompt: + # restore below seeds context and the loop then prompts for + # the next turn via InputCallback. Leaving messages None here + # lets the restore block fill it. + if not self._has_restorable_history(): + from ms_agent.command.interactive import \ + InteractiveSession + session = InteractiveSession( + self._get_command_router(), + source='tui' if self._input_source is not None + else 'cli', + input_source=self._input_source, + event_sink=self._event_sink) + turn = await session.run_turn( + messages=None, runtime=self.runtime) + if turn.action == 'quit': + # Exited at the interactive prompt without a task. + self.runtime.should_stop = True + await self.cleanup_tools() + return + messages = turn.text else: # Non-interactive with no task: accept piped stdin as the # query; otherwise fail clearly instead of blocking input(). @@ -1694,7 +1925,16 @@ async def run_loop(self, messages: Union[List[Message], str], # compression). This is the canonical history for the round, so # it must run before the per-round augmentations below. if self.context_assembler is not None and self.runtime.round > 0: + # Detect real compaction via last_consolidated advancing + # (assemble() only advances it when a strategy consolidated + # the window — see ContextAssembler.assemble). + _lc_before = (self.session_log.last_consolidated + if self.session_log is not None else 0) messages = self.context_assembler.assemble() + if (self._event_sink is not None + and self.session_log is not None + and self.session_log.last_consolidated > _lc_before): + self._event_sink.emit(ContextCompacted()) messages = self._apply_pending_rollback(messages) if self.task_manager is not None: @@ -1758,6 +1998,9 @@ async def run_loop(self, messages: Union[List[Message], str], import traceback logger.warning(traceback.format_exc()) + if self._event_sink is not None: + self._event_sink.emit( + ErrorRaised(message=f'{type(e).__name__}: {e}')) if hasattr(self.config, 'help'): logger.error( f'[{self.tag}] Runtime error, please follow the instructions:\n\n {self.config.help}' diff --git a/ms_agent/callbacks/input_callback.py b/ms_agent/callbacks/input_callback.py index a8e1f6d09..771516994 100644 --- a/ms_agent/callbacks/input_callback.py +++ b/ms_agent/callbacks/input_callback.py @@ -31,13 +31,19 @@ def __init__( self, config: DictConfig, command_router: Optional['CommandRouter'] = None, + input_source: object = None, + event_sink: object = None, ): super().__init__(config) if command_router is None: # Fallback for standalone / test instantiation. In normal CLI use # the agent injects its own router so there is a single instance. command_router = self._build_default_router() - self._session = InteractiveSession(command_router) + self._session = InteractiveSession( + command_router, + source='tui' if input_source is not None else 'cli', + input_source=input_source, + event_sink=event_sink) @staticmethod def _build_default_router() -> 'CommandRouter': diff --git a/ms_agent/cli/cli.py b/ms_agent/cli/cli.py index b43b4fa4f..6e371bb63 100644 --- a/ms_agent/cli/cli.py +++ b/ms_agent/cli/cli.py @@ -7,6 +7,7 @@ from ms_agent.cli.cron import CronCMD from ms_agent.cli.plugin import PluginCMD from ms_agent.cli.run import RunCMD +from ms_agent.cli.tui import TuiCMD from ms_agent.cli.ui import UICMD @@ -28,6 +29,7 @@ def run_cmd(): ACPProxyCmd.define_args(subparsers) ACPRegistryCmd.define_args(subparsers) RunCMD.define_args(subparsers) + TuiCMD.define_args(subparsers) AppCMD.define_args(subparsers) UICMD.define_args(subparsers) CronCMD.define_args(subparsers) diff --git a/ms_agent/cli/run.py b/ms_agent/cli/run.py index ec72e401f..7f9f2cfbc 100644 --- a/ms_agent/cli/run.py +++ b/ms_agent/cli/run.py @@ -260,7 +260,6 @@ def _execute_with_config(self): if tl is None or not OmegaConf.is_config(tl): localsearch_config = { 'paths': paths, - 'work_path': './.sirchmunk', 'mode': 'FAST', } config.tools['localsearch'] = OmegaConf.create( diff --git a/ms_agent/cli/tui.py b/ms_agent/cli/tui.py new file mode 100644 index 000000000..8831d070f --- /dev/null +++ b/ms_agent/cli/tui.py @@ -0,0 +1,96 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +import argparse + +from .base import CLICommand + + +def subparser_func(args): + """Factory for the `tui` sub-command.""" + return TuiCMD(args) + + +class TuiCMD(CLICommand): + """Terminal UI: a lightweight REPL over the LLMAgent lifecycle. + + Note: `ui` is taken by the (gradio) webui, so this command is `tui`. + """ + + name = 'tui' + + def __init__(self, args): + self.args = args + + @staticmethod + def define_args(parsers: argparse.ArgumentParser): + parser: argparse.ArgumentParser = parsers.add_parser(TuiCMD.name) + parser.add_argument( + '--config', + type=str, + default=None, + help='Path to an agent.yaml (or a task dir / modelscope id). ' + 'Defaults to the built-in agent.yaml.') + parser.add_argument( + '--work-dir', + '--work_dir', + dest='work_dir', + type=str, + default=None, + help='Working/project directory (== output_dir). Defaults to the ' + 'current directory. Framework internals go to /.ms_agent/; ' + 'sessions are global under ~/.ms_agent/projects/.') + parser.add_argument( + '--env', + type=str, + default=None, + help='Path to a .env file (defaults to ./.env).') + parser.add_argument( + '--permission_mode', + type=str, + default='restricted', + choices=['auto', 'strict', 'restricted', 'interactive'], + help='Permission mode for tool calls. Default `restricted` so ' + 'non-whitelisted tools ask for confirmation.') + parser.add_argument( + '--trust_remote_code', + action='store_true', + help='Allow loading external code referenced by the config.') + parser.add_argument( + '--mcp-server-file', + '--mcp_server_file', + dest='mcp_server_file', + type=str, + default=None, + help='Path to an MCP servers JSON file ({"mcpServers": {...}}) — ' + 'stdio / sse / streamable_http servers to connect for this session.') + parser.add_argument( + '--emit-events', + '--emit_events', + dest='emit_events', + type=str, + default=None, + help='Also append every structured AgentEvent as JSON lines to ' + 'this file — the exact wire payloads a WebUI backend would ' + 'forward. Use it to inspect/validate the event contract.') + parser.set_defaults(func=subparser_func) + + def execute(self): + import importlib.resources as importlib_resources + + config = self.args.config + if not config: + # Fall back to the packaged default agent.yaml. + default_config = importlib_resources.files('ms_agent').joinpath( + 'agent', 'agent.yaml') + with importlib_resources.as_file(default_config) as p: + config = str(p) + + from ms_agent.tui.app import main + main( + config, + env_file=self.args.env, + permission_mode=self.args.permission_mode, + trust_remote_code=self.args.trust_remote_code, + work_dir=self.args.work_dir, + emit_events=self.args.emit_events, + mcp_server_file=self.args.mcp_server_file, + ) diff --git a/ms_agent/command/builtin/config_cmds.py b/ms_agent/command/builtin/config_cmds.py index d63adc507..30a7200a3 100644 --- a/ms_agent/command/builtin/config_cmds.py +++ b/ms_agent/command/builtin/config_cmds.py @@ -26,23 +26,30 @@ def _persist_model_to_config(config, new_model: str, service=None): """Persist the model (and optional service) change to the project patch. Writes ``llm.model`` (and ``llm.service`` when a provider switch happened) - into ``/.ms-agent/config.yaml`` rather than mutating the - version-controlled source YAML. ``Config.from_task`` merges this patch back - (patch wins) on the next run, so the override survives without ever touching - the committed config. Returns the patch path on success, or None if no - source directory is known or the write fails. + into ``/.ms_agent/config.yaml`` rather than mutating the + version-controlled source YAML. Anchored to the **work dir** (``output_dir``) + — the project — not the config file's directory, so running a shared or + packaged template config (e.g. ``demos/``) never scatters a ``.ms_agent/`` + next to it. The work-dir patch is merged back (patch wins) on the next run. + Returns the patch path on success, or None if no dir is known / write fails. """ from omegaconf import OmegaConf - local_dir = getattr(config, 'local_dir', None) - if not local_dir: + # Prefer the work dir; fall back to the config's own dir only if unset. + base_dir = (getattr(config, 'output_dir', None) + or getattr(config, 'local_dir', None)) + if not base_dir: return None - patch_dir = os.path.join(str(local_dir), '.ms-agent') + # Write to the new .ms_agent/ dir; migrate an existing legacy + # .ms-agent/config.yaml so a single patch file remains authoritative. + patch_dir = os.path.join(str(base_dir), '.ms_agent') patch_path = os.path.join(patch_dir, 'config.yaml') + legacy_path = os.path.join(str(base_dir), '.ms-agent', 'config.yaml') try: os.makedirs(patch_dir, exist_ok=True) - patch = (OmegaConf.load(patch_path) - if os.path.isfile(patch_path) else OmegaConf.create({})) + source = patch_path if os.path.isfile(patch_path) else legacy_path + patch = (OmegaConf.load(source) + if os.path.isfile(source) else OmegaConf.create({})) OmegaConf.update(patch, 'llm.model', new_model, merge=True) if service: OmegaConf.update(patch, 'llm.service', service, merge=True) diff --git a/ms_agent/command/builtin/context_cmds.py b/ms_agent/command/builtin/context_cmds.py index 0fd7a2a6c..86ba0a704 100644 --- a/ms_agent/command/builtin/context_cmds.py +++ b/ms_agent/command/builtin/context_cmds.py @@ -104,28 +104,50 @@ async def cmd_tools(ctx: CommandContext) -> CommandResult: async def cmd_compact(ctx: CommandContext) -> CommandResult: + """Prune old tool outputs from the live context to free tokens. + + Keeps the most recent tool results in full and elides older large ones in + place (the same idea as ContextAssembler's ToolOutputPruner, applied on + demand). Automatic per-round compaction still runs when session_log + + compaction are configured; this is the manual trigger. + """ messages = ctx.extra.get('messages') if not messages: return CommandResult( type=CommandResultType.MESSAGE, content='No messages available.' ) - try: - from ms_agent.session.context_assembler import ContextAssembler # noqa: F401 - except ImportError: + keep_recent = 3 + prune_threshold = 200 + tool_idxs = [ + i for i, m in enumerate(messages) + if getattr(m, 'role', None) == 'tool' + ] + to_prune = tool_idxs[:-keep_recent] if len(tool_idxs) > keep_recent else [] + + pruned = 0 + freed = 0 + for i in to_prune: + m = messages[i] + content = m.content if isinstance(m.content, str) else str( + m.content or '') + if len(content) > prune_threshold and not content.startswith( + '[compacted'): + placeholder = f'[compacted tool output: {len(content)} chars elided]' + freed += len(content) - len(placeholder) + messages[i].content = placeholder + pruned += 1 + + if pruned == 0: return CommandResult( type=CommandResultType.MESSAGE, - content=( - 'Context compaction not available yet.' - ), + content=(f'Nothing to compact ({len(messages)} messages; no large ' + f'old tool outputs beyond the last {keep_recent}).'), ) - return CommandResult( type=CommandResultType.MESSAGE, - content=( - f'Compaction requested. Current message count: {len(messages)}.\n' - f'(Full implementation available after PR#912 merge)' - ), + content=(f'Compacted {pruned} old tool output(s), ~{freed} chars freed. ' + f'The {keep_recent} most recent are kept in full.'), ) diff --git a/ms_agent/command/interactive.py b/ms_agent/command/interactive.py index 597d7da2f..284b3fcff 100644 --- a/ms_agent/command/interactive.py +++ b/ms_agent/command/interactive.py @@ -33,9 +33,18 @@ class InteractiveTurn: class InteractiveSession: """Drives a single ``>>>`` prompt turn, handling slash commands in a loop.""" - def __init__(self, router: CommandRouter, source: str = 'cli') -> None: + def __init__(self, router: CommandRouter, source: str = 'cli', + input_source: Any = None, event_sink: Any = None) -> None: self._router = router self._source = source + # Async InputSource (ms_agent.ui). Its awaitable read_prompt lets the + # native loop drive a TUI / WebUI without blocking the event loop. When + # None, bare input() is used so CLI behavior is unchanged. + self._input_source = input_source + # AgentEventSink for command output (MESSAGE results): routed as a + # Notice so a TUI renders it on the same channel as everything else. + # When None, plain print() is used (CLI). + self._event_sink = event_sink async def run_turn( self, @@ -56,12 +65,22 @@ async def run_turn( """ while True: try: - query = input('>>> ').strip() + if self._input_source is not None: + query = (await + self._input_source.read_prompt('>>> ')).strip() + else: + query = input('>>> ').strip() except (EOFError, KeyboardInterrupt): return InteractiveTurn(action='quit') if not query: continue if not self._router.is_command(query): + # Echo the user turn on the event channel (fills the ACP + # user_message_chunk gap). A TUI ignores it — the terminal + # already shows the typed line — but a WebUI renders it. + if self._event_sink is not None: + from ms_agent.ui.events import UserMessage + self._event_sink.emit(UserMessage(text=query)) return InteractiveTurn(action='submit', text=query) cmd_name, args = self._router.parse_input(query) @@ -82,10 +101,17 @@ async def run_turn( return InteractiveTurn(action='submit', text=query) if result.type == CommandResultType.QUIT: if result.content: - print(result.content) + self._emit(result.content) return InteractiveTurn(action='quit') if result.type == CommandResultType.SUBMIT_PROMPT: return InteractiveTurn(action='submit', text=result.content) # MESSAGE / MUTATE_STATE: show output and prompt again. if result.content: - print(result.content) + self._emit(result.content) + + def _emit(self, text: str) -> None: + if self._event_sink is not None: + from ms_agent.ui.events import Notice + self._event_sink.emit(Notice(level='info', text=text)) + else: + print(text) diff --git a/ms_agent/config/config.py b/ms_agent/config/config.py index 3385128ea..9994c895c 100644 --- a/ms_agent/config/config.py +++ b/ms_agent/config/config.py @@ -10,12 +10,29 @@ from ms_agent.prompting import apply_prompt_files from ms_agent.utils import get_logger -from ms_agent.config.resolver import ConfigResolver from ..utils.constants import TOOL_PLUGIN_NAME from .env import Env logger = get_logger() +# Ambient shell/system environment variables that must NOT act as config +# overrides. ``_update_config`` matches config leaf keys against env names +# case-insensitively, so without this a config key like ``path`` (e.g. a skill +# source ``sources[].path``) gets silently clobbered by the shell's ``$PATH`` +# (``home``←``HOME``, ``user``←``USER``, ... likewise). Legitimate overrides +# (``OPENAI_API_KEY`` → ``llm.openai_api_key``, ```` substitution, +# explicit ``--key value`` argv) are unaffected — only these ambient names are +# dropped from the env-derived override source. +_SHELL_ENV_BLOCKLIST = frozenset({ + 'PATH', 'HOME', 'USER', 'LOGNAME', 'SHELL', 'PWD', 'OLDPWD', 'TERM', + 'LANG', 'LANGUAGE', 'LC_ALL', 'LC_CTYPE', 'TMPDIR', 'TMP', 'TEMP', + 'EDITOR', 'VISUAL', 'PAGER', 'DISPLAY', 'HOSTNAME', 'HOST', 'MAIL', + 'SHLVL', 'COLUMNS', 'LINES', 'IFS', 'PS1', 'PS2', 'MANPATH', + 'LD_LIBRARY_PATH', 'DYLD_LIBRARY_PATH', 'PYTHONPATH', 'CONDA_PREFIX', + 'CONDA_DEFAULT_ENV', 'VIRTUAL_ENV', 'SSH_AUTH_SOCK', 'COLORTERM', + 'TERM_PROGRAM', 'TERM_SESSION_ID', 'XPC_SERVICE_NAME', +}) + class ConfigLifecycleHandler: @@ -90,19 +107,22 @@ def from_task(cls, f'Cannot find any valid config file in {config_dir_or_id}, ' f'supported configs are: {Config.supported_config_names}') envs = Env.load_env(env) + # Drop ambient shell vars so they can't clobber same-named config keys + # (e.g. skills sources[].path <- $PATH). See _SHELL_ENV_BLOCKLIST. + envs = { + k: v + for k, v in envs.items() if k.upper() not in _SHELL_ENV_BLOCKLIST + } cls._update_config(config, envs) _dict_config = cls.parse_args() cls._update_config(config, _dict_config) config.local_dir = config_dir_or_id config.name = name - # Merge the project-level config patch (/.ms-agent/config.yaml) - # written by interactive overrides such as /model. The patch wins over - # the committed YAML so a user override beats the project default, while - # the source file itself is never mutated. - if isinstance(config, DictConfig): - patch = ConfigResolver()._load_project_patch(config_dir_or_id) - if patch is not None: - config = OmegaConf.merge(config, patch) + # The project-level config patch (persisted /model overrides etc.) is + # merged by the agent from /.ms_agent/config.yaml, anchored to + # the work dir — NOT here from the config file's own directory, which + # would leak/scatter overrides when a shared or packaged template config + # is run from a different working directory. See BaseAgent.__init__. config = cls.fill_missing_fields(config) # Prompt files: resolve config.prompt.system from prompts/ directory # if user didn't specify inline prompt.system. @@ -122,6 +142,22 @@ def fill_missing_fields(config: DictConfig) -> DictConfig: config.callbacks = ListConfig([]) return config + @staticmethod + def safe_get_config(config: Union[DictConfig, ListConfig], + dotted_path: str, + default: Any = None) -> Any: + """Safely read a dotted config path, returning ``default`` if missing. + + Example:: + + Config.safe_get_config(cfg, 'tools.file_system.include', []) + """ + try: + value = OmegaConf.select(config, dotted_path, default=default) + except Exception: + return default + return value if value is not None else default + @staticmethod def is_workflow(config: DictConfig) -> bool: assert config.name is not None, 'Cannot find a valid name in this config' @@ -132,16 +168,28 @@ def is_workflow(config: DictConfig) -> bool: @staticmethod def parse_args() -> Dict[str, Any]: - arg_parser = argparse.ArgumentParser() - args, unknown = arg_parser.parse_known_args() - _dict_config = {} - if unknown: - for idx in range(1, len(unknown) - 1, 2): - key = unknown[idx] - value = unknown[idx + 1] - assert key.startswith( - '--'), f'Parameter not correct: {unknown}' - _dict_config[key[2:]] = value + """Best-effort extraction of ad-hoc ``--key value`` overrides from argv. + + This is called by :meth:`from_task` for every config load, including + embedded / non-CLI contexts (pytest, WebUI/Server, TUI). It therefore + must never raise on argv shapes it does not recognize: it scans for + well-formed ``--key value`` pairs and silently ignores everything else + (subcommands, pytest flags, positional test paths, ``-x`` short flags). + Unknown override keys are harmless downstream — ``_update_config`` only + replaces config paths that already exist. + """ + arg_parser = argparse.ArgumentParser(add_help=False) + _, unknown = arg_parser.parse_known_args() + _dict_config: Dict[str, Any] = {} + idx = 0 + while idx < len(unknown): + key = unknown[idx] + if (key.startswith('--') and len(key) > 2 and idx + 1 < len(unknown) + and not unknown[idx + 1].startswith('--')): + _dict_config[key[2:]] = unknown[idx + 1] + idx += 2 + else: + idx += 1 return _dict_config @staticmethod diff --git a/ms_agent/config/mcp_manager.py b/ms_agent/config/mcp_manager.py index 9c8aabec9..47360dca9 100644 --- a/ms_agent/config/mcp_manager.py +++ b/ms_agent/config/mcp_manager.py @@ -43,7 +43,8 @@ def global_mcp_path(self) -> Path: def project_mcp_path(self) -> Path: if self.project_root is None: raise ValueError('project_root is required for project scope') - return self.project_root / '.ms-agent' / 'mcp.json' + from ms_agent.project.paths import project_internal_file + return project_internal_file(self.project_root, 'mcp.json') def _ensure_dir(self, path: Path) -> None: path.parent.mkdir(parents=True, exist_ok=True) diff --git a/ms_agent/config/model_settings.py b/ms_agent/config/model_settings.py new file mode 100644 index 000000000..2f9445a63 --- /dev/null +++ b/ms_agent/config/model_settings.py @@ -0,0 +1,127 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Persistent model-provider settings (WebUI "Model settings" page / 原型图10). + +Manages the ``providers`` / ``default_model`` sections of +``~/.ms_agent/settings.json`` so a UI can add/remove custom providers and +models and pick a default, on top of the read-only built-in registry +(``ms_agent/llm/spec.py``). Read-modify-write is atomic (tmp + rename) and +never clobbers the other settings.json sections (llm, personalization, ...). +""" +from __future__ import annotations + +import json +import os +from pathlib import Path +from typing import Any, Dict, List, Optional + + +class ModelSettingsManager: + """CRUD for custom providers/models + default model in settings.json.""" + + def __init__(self, global_dir: str | Path = '~/.ms_agent') -> None: + self._dir = Path(global_dir).expanduser() + self._path = self._dir / 'settings.json' + + # -- raw settings.json io -- + + def _load_raw(self) -> Dict[str, Any]: + if not self._path.exists(): + return {} + try: + with open(self._path, 'r', encoding='utf-8') as f: + return json.load(f) + except (json.JSONDecodeError, OSError): + return {} + + def _save_raw(self, data: Dict[str, Any]) -> None: + self._dir.mkdir(parents=True, exist_ok=True) + tmp = self._path.with_suffix('.json.tmp') + with open(tmp, 'w', encoding='utf-8') as f: + json.dump(data, f, ensure_ascii=False, indent=2) + os.replace(tmp, self._path) + + # -- providers -- + + def list_custom_providers(self) -> Dict[str, Dict[str, Any]]: + return dict(self._load_raw().get('providers', {}) or {}) + + def list_providers(self) -> List[Dict[str, Any]]: + """Built-in (read-only, from the registry) + custom providers.""" + from ms_agent.llm.spec import get_registry + out: List[Dict[str, Any]] = [] + for spec in get_registry().list_providers(): + out.append({ + 'id': spec.name, + 'name': spec.name, + 'protocol': spec.transport, + 'builtin': True, + 'models': list(getattr(spec, 'keywords', []) or []), + }) + builtin_ids = {p['id'] for p in out} + for pid, p in self.list_custom_providers().items(): + entry = {'id': pid, 'builtin': False, **p} + if pid in builtin_ids: # custom override of a builtin id + entry['overrides_builtin'] = True + out.append(entry) + return out + + def add_provider( + self, + provider_id: str, + *, + name: Optional[str] = None, + protocol: str = 'openai', + api_key: Optional[str] = None, + base_url: Optional[str] = None, + models: Optional[List[str]] = None, + ) -> Dict[str, Any]: + data = self._load_raw() + providers = data.setdefault('providers', {}) + entry = providers.get(provider_id, {}) + entry.update({ + 'name': name or entry.get('name') or provider_id, + 'protocol': protocol, + }) + if api_key is not None: + entry['api_key'] = api_key + if base_url is not None: + entry['base_url'] = base_url + entry['models'] = list(models if models is not None + else entry.get('models', [])) + providers[provider_id] = entry + self._save_raw(data) + return entry + + def remove_provider(self, provider_id: str) -> None: + data = self._load_raw() + if data.get('providers', {}).pop(provider_id, None) is not None: + self._save_raw(data) + + def add_model(self, provider_id: str, model: str) -> None: + data = self._load_raw() + providers = data.setdefault('providers', {}) + entry = providers.setdefault(provider_id, {'name': provider_id, + 'protocol': 'openai', + 'models': []}) + models = entry.setdefault('models', []) + if model not in models: + models.append(model) + self._save_raw(data) + + def remove_model(self, provider_id: str, model: str) -> None: + data = self._load_raw() + entry = data.get('providers', {}).get(provider_id) + if entry and model in entry.get('models', []): + entry['models'].remove(model) + self._save_raw(data) + + # -- default model -- + + def get_default_model(self) -> Optional[str]: + """Returns ``provider/model`` (or bare ``model``), or None.""" + return self._load_raw().get('default_model') + + def set_default_model(self, model: str, provider: Optional[str] = None) -> None: + data = self._load_raw() + data['default_model'] = f'{provider}/{model}' if provider else model + self._save_raw(data) diff --git a/ms_agent/config/resolver.py b/ms_agent/config/resolver.py index edcc32391..13bf6deb3 100644 --- a/ms_agent/config/resolver.py +++ b/ms_agent/config/resolver.py @@ -49,9 +49,28 @@ GLOBAL_MCP_FILE = 'mcp.json' GLOBAL_SKILLS_FILE = 'skills.json' PROJECT_CONFIG_FILE = 'config.yaml' +PROJECT_SETTINGS_LOCAL_FILE = 'settings.local.json' PROJECT_MCP_FILE = 'mcp.json' PROJECT_SKILLS_FILE = 'skills.json' +# New project-local framework dir (underscore, matches ~/.ms_agent) and the +# older hyphenated name kept for read-compat. +PROJECT_INTERNAL_DIR = '.ms_agent' +PROJECT_INTERNAL_DIR_LEGACY = '.ms-agent' + + +def _project_internal_file(project_path: str, filename: str) -> Optional[Path]: + """Return /.ms_agent/, falling back to the legacy + /.ms-agent/. Returns the new-dir path (may not exist) when + neither exists, so callers can treat a missing file uniformly.""" + new = Path(project_path) / PROJECT_INTERNAL_DIR / filename + if new.exists(): + return new + legacy = Path(project_path) / PROJECT_INTERNAL_DIR_LEGACY / filename + if legacy.exists(): + return legacy + return new + class ConfigResolver: """Multi-layer config resolver for server/UI and Playground scenarios.""" @@ -234,14 +253,37 @@ def _load_global_settings(self) -> Optional[DictConfig]: return None def _load_project_patch(self, project_path: str) -> Optional[DictConfig]: - patch_file = Path(project_path) / '.ms-agent' / PROJECT_CONFIG_FILE - if not patch_file.exists(): - return None - try: - return OmegaConf.load(str(patch_file)) - except Exception as e: - logger.warning(f'Failed to load project config patch: {e}') + """Project-level config patch. + + Merges (low -> high priority): + 1. legacy/new ``config.yaml`` (agent schema, direct) + 2. ``settings.local.json`` (settings schema, mapped like the global + settings.json — provider/model/personalization/... ) + Both are optional; new ``.ms_agent/`` is preferred over legacy + ``.ms-agent/``. + """ + layers: List[DictConfig] = [] + + yaml_file = _project_internal_file(project_path, PROJECT_CONFIG_FILE) + if yaml_file and yaml_file.exists(): + try: + layers.append(OmegaConf.load(str(yaml_file))) + except Exception as e: + logger.warning(f'Failed to load project config.yaml: {e}') + + local_file = _project_internal_file( + project_path, PROJECT_SETTINGS_LOCAL_FILE) + if local_file and local_file.exists(): + try: + with open(local_file, 'r', encoding='utf-8') as f: + data = json.load(f) + layers.append(self._settings_to_agent_config(data)) + except (json.JSONDecodeError, OSError) as e: + logger.warning(f'Failed to load settings.local.json: {e}') + + if not layers: return None + return self._merge_layers(layers) # -- merging -- @@ -263,7 +305,7 @@ def _merge_mcp( project_mcp = {} if project_path: project_mcp = self._load_json_safe( - Path(project_path) / '.ms-agent' / PROJECT_MCP_FILE + _project_internal_file(project_path, PROJECT_MCP_FILE) ) if not global_mcp and not project_mcp: @@ -311,7 +353,7 @@ def _merge_skills( project_skills = {} if project_path: project_skills = self._load_json_safe( - Path(project_path) / '.ms-agent' / PROJECT_SKILLS_FILE + _project_internal_file(project_path, PROJECT_SKILLS_FILE) ) if not global_skills and not project_skills: @@ -357,6 +399,18 @@ def _settings_to_agent_config(settings: Dict[str, Any]) -> DictConfig: ] if agent_llm: agent_fields['llm'] = agent_llm + # default_model (from ModelSettingsManager) is a fallback when the llm + # block does not pin a model. Form: "provider/model" or bare "model". + default_model = settings.get('default_model') + if default_model: + agent_llm = agent_fields.setdefault('llm', {}) + if not agent_llm.get('model'): + if '/' in default_model: + prov, mdl = default_model.split('/', 1) + agent_llm.setdefault('service', prov) + agent_llm['model'] = mdl + else: + agent_llm['model'] = default_model if 'output_dir' in settings: agent_fields['output_dir'] = settings['output_dir'] if 'personalization' in settings: diff --git a/ms_agent/config/skills_manager.py b/ms_agent/config/skills_manager.py index cc05ffe0a..02a76f422 100644 --- a/ms_agent/config/skills_manager.py +++ b/ms_agent/config/skills_manager.py @@ -106,7 +106,8 @@ def _global_path(self) -> Path: return self._global_dir / SKILLS_FILE def _project_path(self, project_path: str) -> Path: - return Path(project_path) / PROJECT_META_DIR / SKILLS_FILE + from ms_agent.project.paths import project_internal_file + return project_internal_file(project_path, SKILLS_FILE) def _resolve_path(self, scope: str, project_path: Optional[str]) -> Path: if scope == 'project': diff --git a/ms_agent/hooks/factory.py b/ms_agent/hooks/factory.py index 0327bf202..a5af94d22 100644 --- a/ms_agent/hooks/factory.py +++ b/ms_agent/hooks/factory.py @@ -144,7 +144,8 @@ def build_hook_runtime( )) if 'native' in enabled_sources: - ms_agent_hooks_json = Path(project_path) / '.ms-agent' / 'hooks.json' + from ms_agent.project.paths import project_internal_file + ms_agent_hooks_json = project_internal_file(project_path, 'hooks.json') if ms_agent_hooks_json.is_file(): loaders.append(( 'ms_agent_json', @@ -221,7 +222,11 @@ def _discover_plugin_roots(config: Any, project_path: str) -> list[str]: managed_ids = registry.managed_plugin_ids(project_path) roots: list[str] = [] seen: set[str] = set() - plugins_dir = Path(project_path) / '.ms-agent' / 'plugins' + from ms_agent.project.paths import (project_internal_dir, + legacy_local_internal_dir) + plugins_dir = project_internal_dir(project_path) / 'plugins' + if not plugins_dir.is_dir(): + plugins_dir = legacy_local_internal_dir(project_path) / 'plugins' if plugins_dir.is_dir(): for child in plugins_dir.iterdir(): if not child.is_dir(): diff --git a/ms_agent/hooks/permission_resolve.py b/ms_agent/hooks/permission_resolve.py index b3c775887..0bdaddb09 100644 --- a/ms_agent/hooks/permission_resolve.py +++ b/ms_agent/hooks/permission_resolve.py @@ -59,7 +59,11 @@ async def resolve_hook_permission_decision( permission_enforcer: PermissionEnforcer | None, permission_config: PermissionConfig | None, hook_runtime: HookRuntime | None = None, + force_decision: PermissionDecision | None = None, ) -> PermissionDecision | str: + # ``force_decision`` carries a SafetyGuard ``ask`` (interactive): it must + # reach the handler even when whitelist/memory would otherwise allow, so a + # broad whitelist can't silently bypass a safety confirmation (REVIEW P1-2). if hook_result and hook_result.action == 'deny': return f'Blocked by hook: {hook_result.reason}' @@ -106,5 +110,6 @@ async def resolve_hook_permission_decision( ) if permission_enforcer: - return await permission_enforcer.check(tool_name, args) + return await permission_enforcer.check( + tool_name, args, force_decision=force_decision) return PermissionDecision(action='allow', reason='No permission enforcer') diff --git a/ms_agent/llm/openai_llm.py b/ms_agent/llm/openai_llm.py index d9a9ef927..93afe1586 100644 --- a/ms_agent/llm/openai_llm.py +++ b/ms_agent/llm/openai_llm.py @@ -72,10 +72,18 @@ def __init__( self.model: str = config.llm.model self.max_continue_runs = getattr(config.llm, 'max_continue_runs', None) or MAX_CONTINUE_RUNS + # Resolution: explicit arg -> config field -> env var -> service + # default. The env fallback keeps `service: openai` working with a + # `.env` that only sets OPENAI_BASE_URL/OPENAI_API_KEY (matching the + # provider-router CredentialResolver), instead of silently hitting the + # real OpenAI endpoint. + import os base_url = base_url or getattr( - config.llm, 'openai_base_url', - None) or get_service_config('openai').base_url - api_key = api_key or getattr(config.llm, 'openai_api_key', None) + config.llm, 'openai_base_url', None) or os.environ.get( + 'OPENAI_BASE_URL') or get_service_config('openai').base_url + api_key = api_key or getattr( + config.llm, 'openai_api_key', None) or os.environ.get( + 'OPENAI_API_KEY') self.client = openai.OpenAI( api_key=api_key, diff --git a/ms_agent/llm/transport/openai_compat.py b/ms_agent/llm/transport/openai_compat.py index 7f73cd24d..cba501823 100644 --- a/ms_agent/llm/transport/openai_compat.py +++ b/ms_agent/llm/transport/openai_compat.py @@ -407,7 +407,8 @@ def _stream_continue_generate( message.reasoning_tokens += self._extract_reasoning_tokens( usage) first_run = not messages[-1].to_dict().get(flag, False) - if chunk.choices[0].finish_reason in [ + if self.continue_gen_mode and chunk.choices[ + 0].finish_reason in [ 'length', 'null' ] and (max_runs is None or max_runs != 0): logger.info( @@ -555,7 +556,7 @@ def _continue_generate(self, **kwargs) -> Message: flag = self._continue_flag new_message = self._format_output_message(completion) - if completion.choices[0].finish_reason in [ + if self.continue_gen_mode and completion.choices[0].finish_reason in [ 'length', 'null' ] and (max_runs is None or max_runs != 0): logger.info( diff --git a/ms_agent/memory/default_memory.py b/ms_agent/memory/default_memory.py index 3f76a1bc8..d32b02ad8 100644 --- a/ms_agent/memory/default_memory.py +++ b/ms_agent/memory/default_memory.py @@ -88,9 +88,12 @@ def __init__(self, config: DictConfig): self.run_id: Optional[str] = getattr(memory_config, 'run_id', None) self.compress: Optional[bool] = getattr(config, 'compress', True) self.is_retrieve: Optional[bool] = getattr(config, 'is_retrieve', True) - self.path: Optional[str] = getattr( - memory_config, 'path', - os.path.join(DEFAULT_OUTPUT_DIR, '.default_memory')) + _mem_path = getattr(memory_config, 'path', None) + if not _mem_path: + from ms_agent.project.paths import memory_dir + _work = getattr(config, 'output_dir', None) or DEFAULT_OUTPUT_DIR + _mem_path = str(memory_dir(_work) / 'default') + self.path: Optional[str] = _mem_path self.history_mode = getattr(memory_config, 'history_mode', 'add') self.ignore_roles: List[str] = getattr(memory_config, 'ignore_roles', ['tool', 'system']) diff --git a/ms_agent/memory/unified/orchestrator.py b/ms_agent/memory/unified/orchestrator.py index ee5193b8e..eb979189d 100644 --- a/ms_agent/memory/unified/orchestrator.py +++ b/ms_agent/memory/unified/orchestrator.py @@ -157,11 +157,22 @@ def _parse_config(self, config: Any) -> MemoryConfig: if isinstance(config, MemoryConfig): return config if hasattr(config, "memory") and hasattr(config.memory, "unified_memory"): - return MemoryConfig.from_dict_config(config.memory.unified_memory) + mc = MemoryConfig.from_dict_config(config.memory.unified_memory) + self._default_base_dir(mc, config) + return mc if hasattr(config, "unified_memory"): return MemoryConfig.from_dict_config(config.unified_memory) return MemoryConfig.from_dict_config(config) + @staticmethod + def _default_base_dir(mc: MemoryConfig, config: Any) -> None: + """When base_dir is unset ('.'), root memory under /.ms_agent/memory + instead of the CWD (so MEMORY.md / facts / index don't scatter).""" + if str(getattr(mc, "base_dir", ".")).strip() in (".", "", "./"): + from ms_agent.project.paths import memory_dir + work = getattr(config, "output_dir", None) or "." + mc.base_dir = str(memory_dir(work)) + # =================================================================== # Message conversion helpers diff --git a/ms_agent/permission/config.py b/ms_agent/permission/config.py index 5394fc55d..ae30e7f2a 100644 --- a/ms_agent/permission/config.py +++ b/ms_agent/permission/config.py @@ -116,8 +116,14 @@ def from_dict(cls, d: dict[str, Any], project_root: str | None = None) -> Permis whitelist = tuple(d.get('whitelist', ())) ask_rules = tuple(d.get('ask_rules', ())) user_blacklist = tuple(d.get('blacklist', ())) - blacklist = _DEFAULT_BLACKLIST + tuple( - p for p in user_blacklist if p not in _DEFAULT_BLACKLIST + # The default blacklist blocks network-egress shell commands + # (curl/wget/ssh/...). ``allow_network: true`` opts out of that secure + # default (e.g. to restore legacy "auto = no interception" behavior); + # the user's own blacklist entries still apply. + allow_network = bool(d.get('allow_network', False)) + base_blacklist = () if allow_network else _DEFAULT_BLACKLIST + blacklist = base_blacklist + tuple( + p for p in user_blacklist if p not in base_blacklist ) safety_raw = d.get('safety_rules', {}) diff --git a/ms_agent/permission/enforcer.py b/ms_agent/permission/enforcer.py index c1b788cee..c63fd3c40 100644 --- a/ms_agent/permission/enforcer.py +++ b/ms_agent/permission/enforcer.py @@ -6,6 +6,7 @@ from __future__ import annotations +import asyncio from dataclasses import dataclass from typing import Any, Literal @@ -41,6 +42,24 @@ def __init__( self._handler = handler or AutoPermissionHandler() self._memory = memory or PermissionMemory() self._matcher = PermissionMatcher() + # Parallel tool calls (asyncio.gather in ToolManager.parallel_call_tool) + # would otherwise invoke the interactive handler concurrently — N + # prompts fighting over one terminal deadlocks. Serialize asks with a + # lock created lazily per running loop (the per-turn TUI uses a fresh + # loop each turn, so a single init-time Lock would bind to the wrong one). + self._ask_lock: asyncio.Lock | None = None + self._ask_lock_loop = None + + def _ask_lock_for_loop(self) -> 'asyncio.Lock': + loop = asyncio.get_running_loop() + if self._ask_lock is None or self._ask_lock_loop is not loop: + self._ask_lock = asyncio.Lock() + self._ask_lock_loop = loop + return self._ask_lock + + async def _serialized_ask(self, **kwargs) -> PermissionResponse: + async with self._ask_lock_for_loop(): + return await self._handler.ask(**kwargs) async def check( self, @@ -59,7 +78,7 @@ async def check( if force_decision and force_decision.action == 'ask': suggestions = generate_suggestions(tool_name, tool_args) - response = await self._handler.ask( + response = await self._serialized_ask( tool_name=tool_name, tool_args=tool_args, context=force_decision.reason or '', @@ -86,9 +105,9 @@ async def check( reason='Allowed by remembered permission', ) - # 5. Ask user via handler + # 5. Ask user via handler (serialized against parallel tool calls) suggestions = generate_suggestions(tool_name, tool_args) - response = await self._handler.ask( + response = await self._serialized_ask( tool_name=tool_name, tool_args=tool_args, context='', diff --git a/ms_agent/permission/path_extractors.py b/ms_agent/permission/path_extractors.py index bd1f3c9b9..5cdf4bd11 100644 --- a/ms_agent/permission/path_extractors.py +++ b/ms_agent/permission/path_extractors.py @@ -357,7 +357,7 @@ def _make_filter_entry( def build_extractor_registry() -> dict[str, ExtractorEntry]: - """Build the full 34-command extractor registry.""" + """Build the full 36-command extractor registry.""" registry: dict[str, ExtractorEntry] = {} # Special commands diff --git a/ms_agent/permission/path_validator.py b/ms_agent/permission/path_validator.py index f2fc4a7b5..0eb68c220 100644 --- a/ms_agent/permission/path_validator.py +++ b/ms_agent/permission/path_validator.py @@ -151,12 +151,25 @@ def validate_path( ) -def is_dangerous_removal_path(path: str) -> bool: - """Check if a path is too dangerous for rm/rmdir, even within allowed dirs.""" +def is_dangerous_removal_path( + path: str, extra_patterns: 'tuple[str, ...]' = ()) -> bool: + """Check if a path is too dangerous for rm/rmdir, even within allowed dirs. + + ``extra_patterns`` are configurable (``safety_rules.dangerous_removal_paths``): + each is expanded (``~``) and matched against the normalized path both + literally and as an fnmatch glob (REVIEW P1-8).""" + import fnmatch normalized = _CONSECUTIVE_SLASHES.sub('/', path) if normalized.endswith('/') and len(normalized) > 1: normalized = normalized.rstrip('/') + for pat in extra_patterns or (): + pat_expanded = _CONSECUTIVE_SLASHES.sub( + '/', os.path.expanduser(str(pat))) + if (normalized == pat_expanded.rstrip('/') + or fnmatch.fnmatch(normalized, pat_expanded)): + return True + if normalized == '*': return True if normalized.endswith('/*') or normalized.endswith('\\*'): diff --git a/ms_agent/permission/safety.py b/ms_agent/permission/safety.py index 81a06206a..0f54e8b3f 100644 --- a/ms_agent/permission/safety.py +++ b/ms_agent/permission/safety.py @@ -39,6 +39,7 @@ def __init__( allowed_directories=tuple(self._allowed_dirs), read_only_directories=tuple(self._read_only_dirs), workspace_root=workspace_root, + dangerous_removal_paths=tuple(config.dangerous_removal_paths), ) self._shell_validator = ShellPathValidator( allowed_dirs=self._allowed_dirs, diff --git a/ms_agent/permission/shell_validator.py b/ms_agent/permission/shell_validator.py index e4de48ac5..36e3b46ff 100644 --- a/ms_agent/permission/shell_validator.py +++ b/ms_agent/permission/shell_validator.py @@ -54,6 +54,7 @@ class PathSafetyConfig: allowed_directories: tuple[str, ...] = () read_only_directories: tuple[str, ...] = () workspace_root: str | None = None + dangerous_removal_paths: tuple[str, ...] = () class ShellPathValidator: @@ -287,7 +288,8 @@ def _validate_paths( for path in paths: # Dangerous removal check for rm/rmdir - if cmd_name in ('rm', 'rmdir') and is_dangerous_removal_path(path): + if cmd_name in ('rm', 'rmdir') and is_dangerous_removal_path( + path, self._config.dangerous_removal_paths): return SafetyDecision( action='deny', reason=f'Dangerous removal path: {path}', diff --git a/ms_agent/plugins/config_manager.py b/ms_agent/plugins/config_manager.py index a94d49a16..49dcb1481 100644 --- a/ms_agent/plugins/config_manager.py +++ b/ms_agent/plugins/config_manager.py @@ -34,7 +34,8 @@ def global_plugins_path(self) -> Path: def project_plugins_path(self) -> Path: if self.project_root is None: raise ValueError('project_root is required for project scope') - return self.project_root / PROJECT_META_DIR / PLUGIN_FILE + from ms_agent.project.paths import project_internal_file + return project_internal_file(self.project_root, PLUGIN_FILE) @property def global_plugins_dir(self) -> Path: @@ -44,7 +45,8 @@ def global_plugins_dir(self) -> Path: def project_plugins_dir(self) -> Path: if self.project_root is None: raise ValueError('project_root is required for project scope') - return self.project_root / PROJECT_META_DIR / 'plugins' + from ms_agent.project.paths import project_internal_dir + return project_internal_dir(self.project_root) / 'plugins' @property def global_plugin_data_root(self) -> Path: diff --git a/ms_agent/plugins/installer.py b/ms_agent/plugins/installer.py index 3b16879e2..a27ef93e4 100644 --- a/ms_agent/plugins/installer.py +++ b/ms_agent/plugins/installer.py @@ -228,7 +228,8 @@ def _target_dir( root = Path(project_path or self.project_root or '') if not str(root): raise ValueError('project_path is required for project plugin install') - return root / '.ms-agent' / 'plugins' / plugin_id + from ms_agent.project.paths import project_internal_dir + return project_internal_dir(root) / 'plugins' / plugin_id return self.global_root / 'plugins' / plugin_id def _ensure_dependencies( diff --git a/ms_agent/plugins/runtime.py b/ms_agent/plugins/runtime.py index f871ee20f..75daf818b 100644 --- a/ms_agent/plugins/runtime.py +++ b/ms_agent/plugins/runtime.py @@ -563,6 +563,8 @@ def _is_managed_plugin_path( (global_root / 'plugins').expanduser().resolve(), ] if project_root is not None: + allowed_roots.append( + (project_root / '.ms_agent' / 'plugins').expanduser().resolve()) allowed_roots.append( (project_root / '.ms-agent' / 'plugins').expanduser().resolve()) for root in allowed_roots: diff --git a/ms_agent/project/manager.py b/ms_agent/project/manager.py index 3dcbec892..f3f9d35a0 100644 --- a/ms_agent/project/manager.py +++ b/ms_agent/project/manager.py @@ -11,9 +11,22 @@ class ProjectManager: """Project CRUD. Pure SDK interface, no IO assumptions.""" PROJECTS_DIR = 'projects' - META_DIR = '.ms-agent' + META_DIR = '.ms_agent' + LEGACY_META_DIR = '.ms-agent' META_FILE = 'project.json' + def _meta_file(self, project_id: str) -> Path: + """Project meta path — new ``.ms_agent`` first, legacy ``.ms-agent`` if + only that exists (read-compat).""" + base = self._projects_root / project_id + new = base / self.META_DIR / self.META_FILE + if new.exists(): + return new + legacy = base / self.LEGACY_META_DIR / self.META_FILE + if legacy.exists(): + return legacy + return new + def __init__(self, base_dir: str = '~/.ms_agent') -> None: self._base = Path(os.path.expanduser(base_dir)) self._projects_root = self._base / self.PROJECTS_DIR @@ -27,7 +40,15 @@ def create( instruction: str = '', memory_enabled: bool = False, memory_backend: str | None = None, + init_workspace: bool = True, ) -> Project: + """Create a new project (Codex "start from scratch"). + + Identity is a random id; ``path`` defaults to the managed location + ``/projects//``. Set ``init_workspace=False`` to skip the + ``/workspace/`` subdir (the runtime writes products directly under + ``output_dir``, so that subdir is optional). + """ project_id = _new_id() if path is None: path = str(self._projects_root / project_id) @@ -41,7 +62,46 @@ def create( memory_enabled=memory_enabled, memory_backend=memory_backend, ) - self._init_project_dirs(project) + self._init_project_dirs(project, init_workspace=init_workspace) + self._save_meta(project) + return project + + def open_folder( + self, + path: str, + name: str | None = None, + instruction: str = '', + memory_enabled: bool = False, + memory_backend: str | None = None, + ) -> Project: + """Open an existing directory as a project (Codex "use an existing folder"). + + Unlike :meth:`create`, the project's identity **is the folder**: + ``id = project_key(path)``. Consequences: + + - **Dedup by path** — reopening the same folder returns the same project + (no duplicate), so history is continuous across reopens. + - **No ``workspace/`` pollution** — the agent works in the folder + directly; only the metadata under ``/projects//`` is written + here. The folder's own ``.ms_agent/`` internals are created lazily by + the runtime, not by this call. + """ + from ms_agent.project.paths import project_key + + work_dir = str(Path(os.path.expanduser(path)).resolve()) + project_id = project_key(work_dir) + existing = self.get(project_id) + if existing is not None: + return existing + project = Project( + id=project_id, + name=name or Path(work_dir).name or project_id, + path=work_dir, + instruction=instruction, + memory_enabled=memory_enabled, + memory_backend=memory_backend, + ) + self._init_project_dirs(project, init_workspace=False) self._save_meta(project) return project @@ -59,7 +119,7 @@ def list(self) -> list[Project]: for entry in sorted(self._projects_root.iterdir()): if not entry.is_dir(): continue - meta_file = entry / self.META_DIR / self.META_FILE + meta_file = self._meta_file(entry.name) if meta_file.exists(): store = JSONFileStore(meta_file) try: @@ -98,10 +158,10 @@ def get_default_project(self) -> Project: # -- internal -- def _ensure_default_project(self) -> None: - default_path = self._projects_root / DEFAULT_PROJECT_ID - meta_file = default_path / self.META_DIR / self.META_FILE + meta_file = self._meta_file(DEFAULT_PROJECT_ID) if meta_file.exists(): return + default_path = self._projects_root / DEFAULT_PROJECT_ID project = Project( id=DEFAULT_PROJECT_ID, name='Default', @@ -110,18 +170,23 @@ def _ensure_default_project(self) -> None: self._init_project_dirs(project) self._save_meta(project) - def _init_project_dirs(self, project: Project) -> None: + def _init_project_dirs( + self, project: Project, init_workspace: bool = True + ) -> None: project_dir = self._projects_root / project.id (project_dir / self.META_DIR).mkdir(parents=True, exist_ok=True) - (project_dir / self.META_DIR / 'sessions').mkdir(exist_ok=True) + # Sessions live at /sessions (flat), matching SessionManager + # (paths.py) — no meta-nested sessions dir. + (project_dir / 'sessions').mkdir(exist_ok=True) project_path = Path(project.path) project_path.mkdir(parents=True, exist_ok=True) - (project_path / 'workspace').mkdir(exist_ok=True) + # The workspace/ subdir is optional (the runtime writes under output_dir + # directly). Skip it for open_folder so an existing folder stays clean. + if init_workspace: + (project_path / 'workspace').mkdir(exist_ok=True) def _meta_store(self, project_id: str) -> JSONFileStore: - return JSONFileStore( - self._projects_root / project_id / self.META_DIR / self.META_FILE - ) + return JSONFileStore(self._meta_file(project_id)) def _save_meta(self, project: Project) -> None: project_dir = self._projects_root / project.id diff --git a/ms_agent/project/paths.py b/ms_agent/project/paths.py new file mode 100644 index 000000000..ed0431d8f --- /dev/null +++ b/ms_agent/project/paths.py @@ -0,0 +1,159 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Canonical on-disk layout (Claude Code / Codex aligned). + +Single source of truth for where things live, so the runtime agent, the M1 +project layer, and a future WebUI/Server all agree: + +- **Working / project directory** == ``output_dir`` (one concept). Work products + may be written anywhere under it, like any coding agent. +- **Framework internals** live under ``/.ms_agent/`` (memory, snapshots, + project-level settings), collapsed into one namespace. +- **Session logs live globally**, decoupled from the work dir: + ``~/.ms_agent/projects//.jsonl`` where ```` is the escaped + absolute path of the work dir (CC-style, zero-config per directory). + +Legacy locations (``/sessions``, ``/.ms-agent``, +``/.ms_agent_snapshots``) are still *read* for back-compat. +""" +from __future__ import annotations + +import os +import re +from pathlib import Path + +INTERNAL_DIR_NAME = '.ms_agent' # project-local framework dir +LEGACY_INTERNAL_DIR_NAME = '.ms-agent' # older hyphenated name (read-compat) + + +def global_home() -> Path: + """The global ms-agent home. Read dynamically so ``MS_AGENT_HOME`` can be + redirected (e.g. per-test) without re-importing.""" + return Path(os.environ.get('MS_AGENT_HOME', '~/.ms_agent')).expanduser() + + +# Back-compat module attribute; prefer global_home(). +GLOBAL_HOME = global_home() + + +def _abs(work_dir: str | Path) -> Path: + return Path(work_dir).expanduser().resolve() + + +def project_key(work_dir: str | Path) -> str: + """CC-style stable key from the work dir's absolute path. + + ``/Users/x/proj`` -> ``-Users-x-proj``. Readable and directory-unique. + """ + abs_path = str(_abs(work_dir)) + key = re.sub(r'[^A-Za-z0-9._-]+', '-', abs_path) + return key.strip('-') or 'root' + + +def global_projects_root() -> Path: + return global_home() / 'projects' + + +def global_project_dir(work_dir: str | Path) -> Path: + """``~/.ms_agent/projects//`` — where sessions (and later project + metadata) for this work dir live.""" + return global_projects_root() / project_key(work_dir) + + +def global_sessions_dir(work_dir: str | Path) -> Path: + """Session logs live flat directly under the global project dir.""" + return global_project_dir(work_dir) + + +def local_internal_dir(work_dir: str | Path) -> Path: + """``/.ms_agent/`` — framework internals for this project.""" + return _abs(work_dir) / INTERNAL_DIR_NAME + + +def legacy_local_internal_dir(work_dir: str | Path) -> Path: + return _abs(work_dir) / LEGACY_INTERNAL_DIR_NAME + + +def snapshots_dir(work_dir: str | Path) -> Path: + """``/.ms_agent/snapshots`` (was ``/.ms_agent_snapshots``).""" + return local_internal_dir(work_dir) / 'snapshots' + + +def memory_dir(work_dir: str | Path) -> Path: + """``/.ms_agent/memory`` (was ``/.memory`` / ``.default_memory``).""" + return local_internal_dir(work_dir) / 'memory' + + +# ── generic + specific work-local internal subdirs ────────────────────────── +# Everything the framework writes for its own bookkeeping goes under one of +# these, so a work dir only ever contains the user's artifacts plus a single +# ``.ms_agent/`` namespace (no scattered ms_agent.log / .memory / .index / …). + +def internal_subdir(work_dir: str | Path, *parts: str) -> Path: + """``/.ms_agent/``.""" + return local_internal_dir(work_dir).joinpath(*parts) + + +def artifacts_dir(work_dir: str | Path) -> Path: + """Large tool-output spill (was ``/.ms_agent_artifacts``).""" + return internal_subdir(work_dir, 'artifacts') + + +def index_dir(work_dir: str | Path) -> Path: + """Search / filesystem / code indexes (was ``/.index``).""" + return internal_subdir(work_dir, 'index') + + +def locks_dir(work_dir: str | Path) -> Path: + """Lock files (was ``/.locks``).""" + return internal_subdir(work_dir, 'locks') + + +def tmp_dir(work_dir: str | Path) -> Path: + """Ephemeral working files (was ``/.temp``).""" + return internal_subdir(work_dir, 'tmp') + + +def subagents_dir(work_dir: str | Path) -> Path: + """Sub-agent stream logs (was ``/subagents``).""" + return internal_subdir(work_dir, 'subagents') + + +def web_search_dir(work_dir: str | Path) -> Path: + """Web-search payload spill (was ``/web_search_artifacts``).""" + return internal_subdir(work_dir, 'web_search') + + +def search_index_dir(work_dir: str | Path, name: str = 'search') -> Path: + """RAG / sirchmunk indexes (was ``./.sirchmunk`` / ``./llama_index``).""" + return internal_subdir(work_dir, 'search_index', name) + + +def stats_file(work_dir: str | Path, name: str = 'workflow_stats.json') -> Path: + """Token/usage stats (was ``/workflow_stats.json``).""" + return internal_subdir(work_dir, name) + + +def global_logs_dir() -> Path: + """``~/.ms_agent/logs`` — opt-in file logging target (never CWD).""" + return global_home() / 'logs' + + +# ── project-local config dir (writes prefer new, reads fall back to legacy) ── + +def project_internal_dir(project_path: str | Path) -> Path: + """New project-local framework dir for **writes** (``/.ms_agent``).""" + return _abs(project_path) / INTERNAL_DIR_NAME + + +def project_internal_file(project_path: str | Path, filename: str) -> Path: + """Resolve ``/.ms_agent/`` for reads, falling back to the legacy + ``/.ms-agent/`` when only the legacy exists. Returns the new-dir + path when neither exists so callers treat "missing" uniformly. Writers + should use ``project_internal_dir()`` (always the new dir).""" + new = _abs(project_path) / INTERNAL_DIR_NAME / filename + if new.exists(): + return new + legacy = _abs(project_path) / LEGACY_INTERNAL_DIR_NAME / filename + if legacy.exists(): + return legacy + return new diff --git a/ms_agent/project/session.py b/ms_agent/project/session.py index f7a35e52d..4189b303b 100644 --- a/ms_agent/project/session.py +++ b/ms_agent/project/session.py @@ -26,9 +26,21 @@ class SessionManager: def __init__(self, project: Project) -> None: self._project = project + # Sessions live globally under ~/.ms_agent/projects//sessions, + # co-located with the runtime SessionLog root (paths.py), not inside the + # project's work dir. Legacy /.ms-agent/sessions is migrated + # forward once if present. + from ms_agent.project.paths import global_projects_root self._sessions_dir = ( - Path(project.path) / '.ms-agent' / 'sessions' + global_projects_root() / project.id / 'sessions' ) + legacy = Path(project.path) / '.ms-agent' / 'sessions' + if legacy.is_dir() and not self._sessions_dir.exists(): + try: + import shutil + shutil.copytree(legacy, self._sessions_dir) + except Exception: + pass self._sessions_dir.mkdir(parents=True, exist_ok=True) @property diff --git a/ms_agent/project/workspace.py b/ms_agent/project/workspace.py index 1e289d8fd..d6c192d94 100644 --- a/ms_agent/project/workspace.py +++ b/ms_agent/project/workspace.py @@ -85,6 +85,57 @@ def import_path(self, source: str) -> None: else: shutil.copy2(src, dest) + # -- binary / HTTP transfer (WebUI upload & packaged download) -- + + def read_bytes(self, rel_path: str) -> bytes: + target = self._resolve_safe(rel_path) + if not target.is_file(): + raise FileNotFoundError(f'Not a file: {rel_path}') + return target.read_bytes() + + def write_bytes(self, rel_path: str, data: bytes) -> None: + target = self._resolve_safe(rel_path) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(data) + + def save_upload(self, filename: str, stream, subdir: str = '.') -> str: + """HTTP upload: write a binary stream (file-like or bytes) into the + workspace. ``filename`` is basename-only (dir components stripped); + returns the stored path relative to the workspace root.""" + rel = str(Path(subdir) / Path(filename).name) + target = self._resolve_safe(rel) + target.parent.mkdir(parents=True, exist_ok=True) + if hasattr(stream, 'read'): + with open(target, 'wb') as f: + shutil.copyfileobj(stream, f) + else: + target.write_bytes(bytes(stream)) + return str(target.relative_to(self._root)) + + def zip_download(self, rel_path: str = '.') -> bytes: + """Package a workspace file/dir into a zip and return the bytes.""" + import io + import zipfile + target = self._resolve_safe(rel_path) + if not target.exists(): + raise FileNotFoundError(f'Path not found: {rel_path}') + buf = io.BytesIO() + with zipfile.ZipFile(buf, 'w', zipfile.ZIP_DEFLATED) as zf: + if target.is_file(): + zf.write(target, target.name) + else: + for p in sorted(target.rglob('*')): + if p.is_file(): + # Skip entries that resolve outside the workspace: a + # planted symlink (e.g. -> /etc/passwd) would otherwise + # be followed by is_file() and packaged (info leak). + try: + p.resolve().relative_to(self._root) + except ValueError: + continue + zf.write(p, p.relative_to(target)) + return buf.getvalue() + def _resolve_safe(self, rel_path: str) -> Path: target = (self._root / rel_path).resolve() # Use relative_to() rather than str.startswith(): the latter would diff --git a/ms_agent/rag/llama_index_rag.py b/ms_agent/rag/llama_index_rag.py index 129fda95f..990edbfd4 100644 --- a/ms_agent/rag/llama_index_rag.py +++ b/ms_agent/rag/llama_index_rag.py @@ -33,7 +33,12 @@ def __init__(self, config: DictConfig): self.chunk_size = getattr(config.rag, 'chunk_size', 512) self.chunk_overlap = getattr(config.rag, 'chunk_overlap', 50) self.retrieve_only = getattr(config.rag, 'retrieve_only', False) - self.storage_dir = getattr(config.rag, 'storage_dir', './llama_index') + _rag_store = getattr(config.rag, 'storage_dir', None) + if not _rag_store: + from ms_agent.project.paths import search_index_dir + _rag_store = str( + search_index_dir(getattr(config, 'output_dir', None) or '.', 'rag')) + self.storage_dir = _rag_store self._validate_requirements() self._setup_embedding_model(config) diff --git a/ms_agent/skill/catalog.py b/ms_agent/skill/catalog.py index 90137fc69..7ad9a4ec3 100644 --- a/ms_agent/skill/catalog.py +++ b/ms_agent/skill/catalog.py @@ -79,7 +79,8 @@ def _download_skill_zip(skill_id: str, local_dir: str) -> str: if _candidate.exists(): BUILTIN_SKILLS_DIR = _candidate -USER_SKILLS_DIR = Path.home() / ".ms_agent" / "skills" +from ms_agent.project.paths import global_home as _global_home +USER_SKILLS_DIR = _global_home() / "skills" class SkillCatalog: diff --git a/ms_agent/tools/agent_tool.py b/ms_agent/tools/agent_tool.py index 51fbfd121..5150f9571 100644 --- a/ms_agent/tools/agent_tool.py +++ b/ms_agent/tools/agent_tool.py @@ -1283,8 +1283,8 @@ def _save_transcript(self, messages: Any, if not isinstance(messages, list) or not agent_tag: return try: - output_dir = getattr(self.config, 'output_dir', './output') - subagents_dir = os.path.join(output_dir, 'subagents') + output_dir = getattr(self.config, 'output_dir', None) or '.' + subagents_dir = os.path.join(output_dir, '.ms_agent', 'subagents') os.makedirs(subagents_dir, exist_ok=True) path = os.path.join(subagents_dir, f'{agent_tag}.jsonl') with open(path, 'w', encoding='utf-8') as f: diff --git a/ms_agent/tools/audio_generator/audio_gen.py b/ms_agent/tools/audio_generator/audio_gen.py index 2b533c08b..14067e1dd 100644 --- a/ms_agent/tools/audio_generator/audio_gen.py +++ b/ms_agent/tools/audio_generator/audio_gen.py @@ -9,7 +9,7 @@ class AudioGenerator(ToolBase): def __init__(self, config): super().__init__(config) - self.temp_dir = os.path.join(self.output_dir, '.temp', + self.temp_dir = os.path.join(self.output_dir, '.ms_agent', 'tmp', 'audio_generator') os.makedirs(self.temp_dir, exist_ok=True) audio_generator = self.config.audio_generator diff --git a/ms_agent/tools/code/local_code_executor.py b/ms_agent/tools/code/local_code_executor.py index 54ed4fc21..15cd47735 100644 --- a/ms_agent/tools/code/local_code_executor.py +++ b/ms_agent/tools/code/local_code_executor.py @@ -60,7 +60,7 @@ async def start(self) -> None: # Ensure dependencies exist before importing. install_package('ipykernel') - install_package('jupyter-client') + install_package('jupyter-client', 'jupyter_client') from jupyter_client import AsyncKernelManager diff --git a/ms_agent/tools/code_server/lsp_code_server.py b/ms_agent/tools/code_server/lsp_code_server.py index ac7f3fbaf..d993c6a31 100644 --- a/ms_agent/tools/code_server/lsp_code_server.py +++ b/ms_agent/tools/code_server/lsp_code_server.py @@ -492,7 +492,7 @@ async def start(self) -> bool: return False # Create workspace data directory for jdtls - workspace_data_dir = Path(self.workspace_dir) / '.jdtls_workspace' + workspace_data_dir = Path(self.workspace_dir) / '.ms_agent' / 'lsp' / 'jdtls' workspace_data_dir.mkdir(exist_ok=True) # Start jdtls @@ -559,7 +559,7 @@ async def connect(self) -> None: def cleanup_lsp_index_dirs(self): cleanup_dirs = [ - os.path.join(self.output_dir, '.jdtls_workspace'), # Java LSP + os.path.join(self.output_dir, '.ms_agent', 'lsp', 'jdtls'), # Java LSP os.path.join(self.output_dir, '.pyright'), # Python LSP (if exists) os.path.join(self.output_dir, 'node_modules', diff --git a/ms_agent/tools/image_generator/image_gen.py b/ms_agent/tools/image_generator/image_gen.py index 3ab8a71df..c74291386 100644 --- a/ms_agent/tools/image_generator/image_gen.py +++ b/ms_agent/tools/image_generator/image_gen.py @@ -9,7 +9,7 @@ class ImageGenerator(ToolBase): def __init__(self, config): super().__init__(config) - self.temp_dir = os.path.join(self.output_dir, '.temp', + self.temp_dir = os.path.join(self.output_dir, '.ms_agent', 'tmp', 'image_generator') os.makedirs(self.temp_dir, exist_ok=True) image_generator = self.config.image_generator diff --git a/ms_agent/tools/search/sirchmunk_search.py b/ms_agent/tools/search/sirchmunk_search.py index cd8aaacf5..90b6a8d56 100644 --- a/ms_agent/tools/search/sirchmunk_search.py +++ b/ms_agent/tools/search/sirchmunk_search.py @@ -84,7 +84,12 @@ def __init__(self, config: DictConfig): str(Path(p).expanduser().resolve()) for p in paths ] - _work_path = rag_config.get('work_path', './.sirchmunk') + _work_path = rag_config.get('work_path', None) + if not _work_path: + from ms_agent.project.paths import search_index_dir + _work_path = str( + search_index_dir(getattr(config, 'output_dir', None) or '.', + 'sirchmunk')) self.work_path: Path = Path(_work_path).expanduser().resolve() self.reuse_knowledge = rag_config.get('reuse_knowledge', True) diff --git a/ms_agent/tools/search/websearch_tool.py b/ms_agent/tools/search/websearch_tool.py index 167bbe3bb..aab578184 100644 --- a/ms_agent/tools/search/websearch_tool.py +++ b/ms_agent/tools/search/websearch_tool.py @@ -625,8 +625,8 @@ def __init__(self, config, **kwargs): getattr(tool_cfg, 'spill_max_inline_chars', 120000) or 120000) if tool_cfg else 120000 self._spill_subdir = str( - getattr(tool_cfg, 'spill_subdir', 'web_search_artifacts') - or 'web_search_artifacts') if tool_cfg else 'web_search_artifacts' + getattr(tool_cfg, 'spill_subdir', '.ms_agent/web_search') + or '.ms_agent/web_search') if tool_cfg else '.ms_agent/web_search' self._spill_preview_chars = int( getattr(tool_cfg, 'spill_preview_chars', 600) or 600) if tool_cfg else 600 diff --git a/ms_agent/tools/tool_manager.py b/ms_agent/tools/tool_manager.py index 10884be22..5a0430db0 100644 --- a/ms_agent/tools/tool_manager.py +++ b/ms_agent/tools/tool_manager.py @@ -458,6 +458,7 @@ async def single_call_tool(self, tool_info: ToolCall): # --- Permission checks --- args_dict = dict(tool_args) if isinstance(tool_args, dict) else {} + safety_force_decision = None if self._safety_guard is not None: from ms_agent.permission.ask_resolver import resolve_ask safety_decision = self._safety_guard.check(tool_name, args_dict) @@ -470,7 +471,12 @@ async def single_call_tool(self, tool_info: ToolCall): if resolved.action == 'ask': if self._permission_enforcer is None: return f'Blocked by safety policy (requires confirmation): {resolved.reason}' - # interactive mode: fall through to enforcer/handler + # interactive mode: force the enforcer to confirm with + # the user; whitelist/memory must not bypass a safety + # ask (REVIEW P1-2). + from ms_agent.permission.enforcer import PermissionDecision + safety_force_decision = PermissionDecision( + action='ask', reason=resolved.reason) # --- PreToolUse hooks --- hook_result = None @@ -497,6 +503,7 @@ async def single_call_tool(self, tool_info: ToolCall): permission_enforcer=self._permission_enforcer, permission_config=self._permission_config, hook_runtime=self._hook_runtime, + force_decision=safety_force_decision, ) if isinstance(perm_out, str): return perm_out diff --git a/ms_agent/tools/video_generator/video_gen.py b/ms_agent/tools/video_generator/video_gen.py index 7578ebf59..dddb7247b 100644 --- a/ms_agent/tools/video_generator/video_gen.py +++ b/ms_agent/tools/video_generator/video_gen.py @@ -9,7 +9,7 @@ class VideoGenerator(ToolBase): def __init__(self, config): super().__init__(config) - self.temp_dir = os.path.join(self.output_dir, '.temp', + self.temp_dir = os.path.join(self.output_dir, '.ms_agent', 'tmp', 'video_generator') os.makedirs(self.temp_dir, exist_ok=True) video_generator = self.config.video_generator diff --git a/ms_agent/tui/__init__.py b/ms_agent/tui/__init__.py new file mode 100644 index 000000000..3ac5d1a2c --- /dev/null +++ b/ms_agent/tui/__init__.py @@ -0,0 +1,20 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Terminal UI (TUI) for ms-agent — a scrolling REPL with a status bar. + +A route-A driver over the native LLMAgent lifecycle: the agent owns one +``run_loop`` per session (auto-compaction, single lifecycle, no per-turn +teardown) while the TUI supplies three UI-agnostic seams from ``ms_agent.ui``: + +* :class:`~ms_agent.tui.renderer.RichEventSink` — renders the structured + ``AgentEvent`` stream with ``rich``. +* :class:`~ms_agent.tui.input.PromptToolkitInput` — awaitable input + status bar. +* :class:`~ms_agent.tui.permission.TUIPermissionHandler` — restricted-tool asks. + +The same seams a WebUI backend consumes, so the TUI validates that contract. +""" +from ms_agent.tui.app import TuiApp, main +from ms_agent.tui.input import PromptToolkitInput +from ms_agent.tui.renderer import RichEventSink +from ms_agent.tui.state import TuiState + +__all__ = ['TuiApp', 'main', 'RichEventSink', 'PromptToolkitInput', 'TuiState'] diff --git a/ms_agent/tui/app.py b/ms_agent/tui/app.py new file mode 100644 index 000000000..5477131f2 --- /dev/null +++ b/ms_agent/tui/app.py @@ -0,0 +1,478 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""TUI application — a route-A driver over the native LLMAgent lifecycle. + +The agent owns one continuous ``run_loop`` per session: a single SessionStart, +automatic context compaction, and no per-turn MCP teardown. The TUI is a thin +driver — it injects three seams and then gets out of the way: + +* ``event_sink`` → :class:`RichEventSink` renders the structured event stream. +* ``input_source`` → :class:`PromptToolkitInput` supplies awaitable prompts. +* ``permission`` → :class:`TUIPermissionHandler` confirms restricted tools. + +Session switching (``/new`` / ``/resume`` / ``/sessions``) happens *between* +lifecycles: the command signals a pending switch and stops the loop; the driver +repoints the agent's session log at the chosen SessionManager session and runs +again. Message persistence and compaction are the agent's job (route A), so the +driver keeps none of the route-B bookkeeping. + +Positioning: single user · one project per work dir · many sessions per project. +""" +from __future__ import annotations + +import asyncio +import logging +import os +from pathlib import Path +from typing import Optional, Tuple + +from omegaconf import OmegaConf +from rich.console import Console +from rich.panel import Panel +from rich.table import Table + +from ms_agent.config import Config +from ms_agent.config.env import Env +from ms_agent.tui.input import PromptToolkitInput +from ms_agent.tui.renderer import RichEventSink +from ms_agent.tui.state import TuiState +from ms_agent.tui.theme import DEFAULT_THEME +from ms_agent.utils.logger import get_logger + +logger = get_logger() + + +class TuiApp: + + def __init__( + self, + config_path: str, + env_file: Optional[str] = None, + permission_mode: Optional[str] = None, + trust_remote_code: bool = False, + work_dir: Optional[str] = None, + emit_events: Optional[str] = None, + mcp_server_file: Optional[str] = None, + ) -> None: + Env.load_dotenv_into_environ(env_file) + self.console = Console() + self.theme = DEFAULT_THEME + self.trust_remote_code = trust_remote_code + self.work_dir = str(Path(work_dir).expanduser().resolve() + if work_dir else Path.cwd().resolve()) + + config = Config.from_task(config_path) + config = self._prepare_config(config, permission_mode, self.work_dir) + self.config = config + + mode = str(getattr(getattr(config, 'permission', None), 'mode', 'auto')) + self.permission_mode = mode + self._model = str( + getattr(getattr(config, 'llm', None), 'model', '') or '') + + # Shared state: renderer writes token usage, the input bar reads it. + self.state = TuiState( + model=self._model, perm=mode, work_dir=self.work_dir) + self.renderer = RichEventSink(self.console, self.state, self.theme) + + # Optionally tee every event to a JSONL file (the exact WebUI wire + # payloads) for contract inspection, without changing what's rendered. + self._jsonl_sink = None + event_sink = self.renderer + if emit_events: + from ms_agent.ui.events import JsonlEventSink, TeeEventSink + self._jsonl_sink = JsonlEventSink(emit_events) + event_sink = TeeEventSink(self.renderer, self._jsonl_sink) + + # Session layer (M1). Work dir == project (CC-aligned): sessions live at + # ~/.ms_agent/projects//sessions. + from ms_agent.project import SessionManager + from ms_agent.project.paths import project_key + from ms_agent.project.types import Project + proj = Project(id=project_key(self.work_dir), + name=Path(self.work_dir).name or 'project', + path=self.work_dir) + self._sm = SessionManager(proj) + self.session = None + + # Bridge managed config files into the runtime (what a WebUI backend + # also does): MCP servers from ~/.ms_agent + /.ms_agent + # mcp.json → mcp_config; skill sources/disabled from skills.json → + # config.skills. Respects enable/disable; --mcp-server-file wins last. + from ms_agent.project.paths import global_home + from ms_agent.tui.managed_config import (merge_skills_into_config, + resolve_mcp_config) + home = str(global_home()) + config = merge_skills_into_config(config, home, self.work_dir) + self.config = config + self._mcp_config = resolve_mcp_config( + home, self.work_dir, mcp_server_file) + + # Build the agent ONCE with the UI seams injected. load_cache is set + # per session by _apply_session (True only on resume). + from ms_agent.agent.llm_agent import LLMAgent + self.agent = LLMAgent( + config, trust_remote_code=trust_remote_code, + event_sink=event_sink, mcp_config=self._mcp_config) + + # Consume the agent's single command router (no duplicate); register the + # TUI session commands and drive input through it (slash completion). + self.router = self.agent._get_command_router() + self._register_session_commands() + self.input = PromptToolkitInput( + self.state, self.router, self.theme, console=self.console) + self.agent._input_source = self.input + + # Always inject the TUI handler — it is only *called* in interactive + # mode, but injecting it unconditionally lets /permission switch into + # interactive at runtime and get real confirmations. + from ms_agent.tui.permission import TUIPermissionHandler + self.agent.set_permission_handler( + TUIPermissionHandler(console=self.console, theme=self.theme)) + + # ('new', None) | ('resume', '<#|id>') | None, set by session commands. + self._pending_switch: Optional[Tuple[str, Optional[str]]] = None + + # -- config shaping -- + + @staticmethod + def _prepare_config(config, permission_mode, work_dir): + OmegaConf.update(config, 'generation_config.stream', True, merge=True) + OmegaConf.update( + config, 'generation_config.stream_output', True, merge=True) + # Show the model's thinking (rendered as a dim collapsed block). Whether + # any reasoning is produced still depends on the model / the config's + # generation_config.extra_body.enable_thinking. + OmegaConf.update( + config, 'generation_config.show_reasoning', True, merge=True) + OmegaConf.update(config, 'output_dir', work_dir, merge=True) + # Route A: the agent owns the session log (enables auto-compaction); + # _apply_session points it at the SessionManager session dir. + OmegaConf.update(config, 'session_log.enabled', True, merge=True) + # Interactive lifecycle regardless of stdin detection. + OmegaConf.update(config, 'interactive', True, merge=True) + # max_chat_round bounds autonomous *steps*; under route A the counter + # accumulates across interactive turns (and restores on resume), so a + # small per-task value would cut a long chat short. Raise it high — the + # user (not a round cap) ends an interactive session. + OmegaConf.update(config, 'max_chat_round', 1000, merge=True) + if permission_mode: + OmegaConf.update( + config, 'permission.mode', permission_mode, merge=True) + # The agent auto-adds InputCallback for interactive runs; drop any + # listed one so restarts across session switches never double-register. + cbs = [c for c in list(getattr(config, 'callbacks', []) or []) + if c != 'input_callback'] + OmegaConf.update(config, 'callbacks', cbs, merge=False) + # Merge the work-dir project patch (e.g. a persisted /model override). + try: + from ms_agent.config.resolver import ConfigResolver + patch = ConfigResolver()._load_project_patch(work_dir) + if patch is not None: + config = OmegaConf.merge(config, patch) + except Exception: + logger.debug('work-dir config patch merge skipped', exc_info=True) + return config + + # -- session commands (registered into the agent's router) -- + + def _register_session_commands(self) -> None: + from ms_agent.command.types import (CommandDef, CommandResult, + CommandResultType) + + async def _sessions(ctx): + self._render_sessions() + return CommandResult(type=CommandResultType.MESSAGE, content='') + + async def _resume(ctx): + self._pending_switch = ('resume', (ctx.args or '').strip()) + if ctx.runtime is not None: + ctx.runtime.should_stop = True + return CommandResult(type=CommandResultType.QUIT, content='') + + async def _new(ctx): + self._pending_switch = ('new', None) + if ctx.runtime is not None: + ctx.runtime.should_stop = True + return CommandResult(type=CommandResultType.QUIT, content='') + + async def _permission(ctx): + arg = (ctx.args or '').strip().lower() + if arg not in ('auto', 'strict', 'restricted', 'interactive'): + return CommandResult( + type=CommandResultType.MESSAGE, + content=(f'permission mode: {self.state.perm}\n' + 'usage: /permission ')) + try: + mode = self.agent.set_permission_mode(arg) + except ValueError as e: + return CommandResult(type=CommandResultType.MESSAGE, + content=str(e)) + self.state.perm = mode + self.permission_mode = mode + return CommandResult(type=CommandResultType.MESSAGE, + content=f'permission mode → {mode}') + + self.router.register( + CommandDef(name='permission', + description='Show or switch permission mode', + category='config', aliases=('mode', )), _permission) + self.router.register( + CommandDef(name='sessions', + description='List sessions in this project', + category='session'), _sessions) + self.router.register( + CommandDef(name='resume', + description='Resume a session: /resume <#|id>', + category='session'), _resume) + self.router.register( + CommandDef(name='new', description='Start a new session', + category='session', aliases=('reset', )), _new) + + # -- session lifecycle -- + + def _apply_session(self, session, resume: bool = False) -> None: + """Point the agent's native session log at this SessionManager session + so the two never diverge (closes the route-B dir split). + + Targets ``self.agent.config`` (not the app's copy): ``read_history`` + reassigns the agent's config object each ``run_loop``, so the app's + reference goes stale after the first lifecycle. ``_init_session_log`` + reads this value before ``read_history`` runs, so setting it here (just + before each run) is what takes effect. + + ``load_cache`` is set per session: ``True`` only when resuming (restore + the SessionLog into context). For a fresh session it MUST be ``False``, + otherwise the legacy output_dir/tag history (``read_history``) would be + loaded when the new SessionLog is still empty — replaying the previous + session's messages. SessionLog is the single source of truth here. + """ + sess_dir = str(self._sm.sessions_dir / session.id) + cfg = self.agent.config + OmegaConf.update(cfg, 'session_log.dir', sess_dir, merge=True) + OmegaConf.update( + cfg, 'session_log.session_key', session.session_key, merge=True) + self.config = cfg # keep the app reference in sync for banners/views + self.session = session + self.state.session_name = session.name + self.agent.load_cache = resume + + def _resume_target(self, arg: str): + sessions = self._sm.list() + if arg.isdigit() and int(arg) < len(sessions): + return sessions[int(arg)] + return next((s for s in sessions if s.id == arg), None) + + def _session_has_history(self, session) -> bool: + try: + return any(m.get('role') == 'user' and m.get('content') + for m in self._sm.get_session_log(session) + .get_all_messages()) + except Exception: + return True # on doubt, keep it + + def _prune_empty_sessions(self) -> None: + """Delete sessions that never received a user turn (leftover empties + from prior launches), so ``/sessions`` stays meaningful.""" + for s in self._sm.list(): + self._prune_if_empty(s) + + def _prune_if_empty(self, session) -> None: + if not self._session_has_history(session): + try: + self._sm.delete(session.id) + except Exception: + pass + + def _name_session_from_log(self) -> None: + """Name a session after its first user line (once), read from its log.""" + if self.session is None or not self.session.name.startswith('Session '): + return + try: + for m in self._sm.get_session_log(self.session).get_all_messages(): + if m.get('role') == 'user' and m.get('content'): + name = str(m['content']).strip().splitlines()[0][:40] + self._sm.update(self.session.id, name=name) + self.session = self._sm.get(self.session.id) or self.session + self.state.session_name = self.session.name + break + except Exception: + pass + + # -- rendering (session views the app owns) -- + + def _banner(self) -> None: + from rich.text import Text + + from ms_agent.utils.constants import MS_AGENT_ASCII + + # Left: the CLI's MS-AGENT wordmark (glyph rows only, frame stripped), + # a blue gradient. Right: the run info — laid out side by side. + glyphs = [ln[1:-1].strip() for ln in MS_AGENT_ASCII.split('\n') + if '█' in ln or '╚═╝' in ln] + width = max(len(g) for g in glyphs) + glyphs = [g.ljust(width) for g in glyphs] # equal width, clean right edge + blues = ['bright_blue', 'blue', 'dodger_blue2', + 'deep_sky_blue1', 'cornflower_blue', 'blue'] + logo = Text() + for i, row in enumerate(glyphs): + logo.append(row + ('\n' if i < len(glyphs) - 1 else ''), + style=f'bold {blues[i % len(blues)]}') + + llm = getattr(self.config, 'llm', None) + tools = list(self.config.tools.keys()) if getattr( + self.config, 'tools', None) else [] + home = os.path.expanduser('~') + wd = self.work_dir.replace(home, '~') if home else self.work_dir + if len(wd) > 38: + wd = '…' + wd[-37:] + info = Table.grid(padding=(0, 2)) + info.add_column(style='blue', justify='right') + info.add_column() + info.add_row('model', + f'{getattr(llm, "service", "?")}/{getattr(llm, "model", "?")}') + info.add_row('tools', ', '.join(tools) or '[dim]none[/]') + info.add_row('perm', f'[yellow]{self.permission_mode}[/]') + info.add_row('dir', f'[dim]{wd}[/]') + info_panel = Panel(info, title='[bold bright_blue]ms-agent tui[/]', + border_style='blue', padding=(0, 2), expand=False) + + banner = Table.grid(padding=(0, 4)) + banner.add_column(no_wrap=True, vertical='middle') # wordmark intact + banner.add_column(vertical='middle') + banner.add_row(logo, info_panel) + self.console.print() + self.console.print(banner) + self.console.print( + ' [dim]/help /sessions /resume /new /quit[/]\n') + + def _render_sessions(self) -> None: + sessions = self._sm.list() + if not sessions: + self.console.print('[dim]no sessions yet[/]') + return + t = Table(show_header=True, header_style='bold', box=None, + padding=(0, 2)) + for c in ('#', 'name', 'id', 'updated', 'model'): + t.add_column(c) + for i, s in enumerate(sessions): + marker = (f'[{self.theme.session_marker}]➤[/]' + if self.session and s.id == self.session.id else str(i)) + t.add_row(marker, s.name, s.id, (s.updated_at or '')[:19], + s.model or '') + self.console.print( + Panel(t, title='sessions', border_style='blue', + subtitle='[dim]/resume <#|id>[/]', expand=False)) + + def _render_history_tail(self, session, n: int = 6) -> None: + try: + msgs = self._sm.get_session_log(session).get_all_messages() + except Exception: + msgs = [] + convo = [m for m in msgs if m.get('role') != 'system'] + for m in convo[-n:]: + who = {'user': '[cyan]❯[/]', 'assistant': '[green]●[/]', + 'tool': '[yellow]▸[/]'}.get(m.get('role'), str(m.get('role'))) + snippet = str(m.get('content') or '')[:200].replace('\n', ' ') + if snippet: + self.console.print(f'{who} {snippet}') + + @staticmethod + def _quiet_logs() -> None: + os.environ.setdefault('LOG_LEVEL', 'ERROR') + if os.environ.get('LOG_LEVEL', 'ERROR').upper() in ( + 'INFO', 'DEBUG', 'WARNING'): + return + lg = logging.getLogger('ms_agent') + lg.setLevel(logging.ERROR) + for h in lg.handlers: + h.setLevel(logging.ERROR) + + # -- main loop (route A: one lifecycle per session) -- + + async def _serve(self) -> None: + self._banner() + self._prune_empty_sessions() # clear leftover empties from prior launches + self.session = self._sm.create(model=self._model or None) + resume = False # a fresh session reads a prompt; a resumed one restores + self.renderer.rule(f'session {self.session.id}', 'green') + while True: + self._pending_switch = None + self._apply_session(self.session, resume=resume) + try: + gen = await self.agent.run(None, stream=True) + async for _ in gen: + pass + except EOFError: + break # Ctrl-D at the prompt exits + except KeyboardInterrupt: + # Ctrl-C interrupts the running turn but keeps the REPL alive. + # Cap the turn with an assistant marker so the resume below + # doesn't re-run the interrupted user prompt. + self.renderer.finalize() + self.console.print('[dim](interrupted — /quit to exit)[/]') + try: + self._sm.get_session_log(self.session).append( + {'role': 'assistant', 'content': '(interrupted)'}) + except Exception: + pass + resume = True + continue + except Exception as e: # noqa: BLE001 — surface, don't crash the REPL + self.renderer.finalize() + logger.warning('TUI lifecycle error', exc_info=True) + self.console.print( + Panel(f'[bold]{type(e).__name__}[/]: {e}', + title='[red]error[/]', border_style='red', + subtitle='[dim]LOG_LEVEL=INFO for details[/]', + expand=False)) + break + self._name_session_from_log() + # Resolve a resume target against the live list before pruning. + switch = self._pending_switch + resume_target = (self._resume_target(switch[1] or '') + if switch and switch[0] == 'resume' else None) + if not (resume_target and resume_target.id == self.session.id): + self._prune_if_empty(self.session) + if switch is None: + break # user quit + kind = switch[0] + if kind == 'new': + self.session = self._sm.create(model=self._model or None) + resume = False + self.renderer.rule(f'new session {self.session.id}', 'green') + elif kind == 'resume': + if resume_target is None: + self.console.print( + '[yellow]session not found[/] — /sessions to list') + resume = True # re-enter (restore) the current session + else: + self.session = resume_target + resume = True + self.renderer.rule( + f'resumed {resume_target.name}', 'green') + self._render_history_tail(resume_target) + self.console.print('[dim]bye[/]') + + def run(self) -> None: + self._quiet_logs() + try: + asyncio.run(self._serve()) + except KeyboardInterrupt: + self.console.print('\n[dim]bye[/]') + finally: + if self._jsonl_sink is not None: + self._jsonl_sink.close() + + +def main( + config_path: str, + env_file: Optional[str] = None, + permission_mode: Optional[str] = None, + trust_remote_code: bool = False, + work_dir: Optional[str] = None, + emit_events: Optional[str] = None, + mcp_server_file: Optional[str] = None, +) -> None: + TuiApp(config_path, env_file=env_file, permission_mode=permission_mode, + trust_remote_code=trust_remote_code, work_dir=work_dir, + emit_events=emit_events, mcp_server_file=mcp_server_file).run() diff --git a/ms_agent/tui/input.py b/ms_agent/tui/input.py new file mode 100644 index 000000000..86806abe0 --- /dev/null +++ b/ms_agent/tui/input.py @@ -0,0 +1,132 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""prompt_toolkit-backed async input for the TUI. + +Implements the ``ms_agent.ui.InputSource`` contract: an awaitable +``read_prompt`` that reads a line via ``PromptSession.prompt_async`` so the +native agent loop can await input without blocking the event loop. Adds a +persistent bottom status bar (model · tokens · perm · session) and slash-command +completion sourced from the agent's ``CommandRouter`` (scoped to ``tui``). + +Degrades gracefully: when prompt_toolkit or a TTY is unavailable (pipes, CI), +falls back to blocking ``input()`` in an executor, so the TUI stays scriptable. +""" +from __future__ import annotations + +import asyncio +import sys +from typing import TYPE_CHECKING, Optional + +from ms_agent.tui.state import TuiState +from ms_agent.tui.theme import DEFAULT_THEME, Theme + +if TYPE_CHECKING: + from ms_agent.command.router import CommandRouter + + +def slash_matches(router, text: str): + """Yield ``(name, description)`` for /command completions of ``text``. + + Only fires when the line starts with ``/`` (so file paths and normal text + don't trigger a menu). Scoped to the ``tui`` command source; de-dupes names + and aliases. Kept dependency-free so it's unit-testable without a terminal. + """ + if router is None or not text.startswith('/'): + return + word = text[1:].lower() + seen = set() + for cmds in router.list_commands('tui').values(): + for c in cmds: + for nm in (c.name, *getattr(c, 'aliases', ())): + if nm.lower().startswith(word) and nm not in seen: + seen.add(nm) + yield nm, c.description + + +class PromptToolkitInput: + """Async input source with a status bar and slash completion.""" + + def __init__(self, state: TuiState, + router: Optional['CommandRouter'] = None, + theme: Theme = DEFAULT_THEME, + console: object = None) -> None: + self._state = state + self._router = router + self._theme = theme + self._console = console # rich Console, for the pre-prompt divider + self._session = self._build_session() + + def _build_session(self): + # Only build a prompt_toolkit session for a real terminal. Constructing + # one under a pipe/CI emits a "not a terminal" warning; the executor + # input() fallback in read_prompt handles those cases cleanly. + if not sys.stdin.isatty(): + return None + try: + from prompt_toolkit import PromptSession + from prompt_toolkit.completion import Completer, Completion + from prompt_toolkit.history import InMemoryHistory + from prompt_toolkit.styles import Style + + router = self._router + + class _SlashCompleter(Completer): + def get_completions(self, document, complete_event): + text = document.text_before_cursor + for name, desc in slash_matches(router, text): + yield Completion('/' + name, start_position=-len(text), + display=f'/{name}', display_meta=desc) + + from prompt_toolkit.formatted_text import HTML + + style = Style.from_dict({ + 'completion-menu': 'bg:#1c1c1c', + 'completion-menu.completion': 'bg:#1c1c1c #d0d0d0', + 'completion-menu.completion.current': 'bg:#00afd7 #000000 bold', + 'completion-menu.meta.completion': 'bg:#1c1c1c #6c6c6c', + 'completion-menu.meta.completion.current': 'bg:#0087af #e4e4e4', + 'scrollbar.background': 'bg:#303030', + 'scrollbar.button': 'bg:#00afd7', + # a calm dim status line, not the default reversed bar + 'bottom-toolbar': 'noreverse bg:default #808080', + 'bottom-toolbar.text': 'noreverse bg:default #808080', + }) + return PromptSession( + history=InMemoryHistory(), + completer=_SlashCompleter(), + complete_while_typing=True, + bottom_toolbar=self._toolbar, + placeholder=HTML( + 'Type a message' + ' · / for commands · ↑ history'), + style=style) + except Exception: + return None + + def _toolbar(self): + s = self._state + parts = [] + if s.model: + parts.append(s.model) + parts.append(f'{s.total_tokens} tok') + if s.perm: + parts.append(s.perm) + if s.session_name: + parts.append(s.session_name) + return ' ' + ' · '.join(parts) + + def _message(self): + from prompt_toolkit.formatted_text import HTML + return HTML(f'{self._theme.prompt_symbol} ') + + async def read_prompt(self, prompt: str = '❯ ') -> str: + # The TUI owns its prompt presentation; the caller's ``prompt`` arg is + # only a fallback hint for the non-interactive path. + if self._session is not None and sys.stdin.isatty(): + # A subtle full-width divider separates the input from the + # conversation above (Claude Code style). + if self._console is not None: + from rich.rule import Rule + self._console.print(Rule(style='grey30')) + return await self._session.prompt_async(self._message()) + loop = asyncio.get_running_loop() + return await loop.run_in_executor(None, input, prompt) diff --git a/ms_agent/tui/managed_config.py b/ms_agent/tui/managed_config.py new file mode 100644 index 000000000..16fdf5d7d --- /dev/null +++ b/ms_agent/tui/managed_config.py @@ -0,0 +1,115 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Bridge the managed config files into the agent runtime. + +`MCPConfigManager` / `SkillsConfigManager` persist MCP servers and skill sources +to ``~/.ms_agent/{mcp,skills}.json`` (global) and ``/.ms_agent/*.json`` +(project) — the UI truth. But the agent runtime doesn't read those files: it +takes MCP via the ``mcp_config`` kwarg and skills via ``config.skills``. This +module performs the translation (read managed files → feed the runtime), which +is exactly what a WebUI backend must also do, so it doubles as the reference. + +Semantics (respecting the runtime's existing same-name-replace merge): + * MCP — global + project merged (project wins by name), **disabled dropped**, + UI meta fields stripped; an explicit ``--mcp-server-file`` wins last. + * Skill — managed sources appended to ``config.skills.sources`` (catalog dedups + by skill_id), managed ``disabled`` unioned into ``config.skills.disabled``. +""" +from __future__ import annotations + +import json +import os +from typing import Any, Dict, Optional + +from omegaconf import OmegaConf + +# UI/meta fields on a managed MCP entry that must not reach the runtime server +# config (the runtime just connects whatever is in mcpServers). +_MCP_META = frozenset({ + 'enabled', 'meta', 'source', '_scope', 'mcp', 'implementation', + 'trust_remote_code', '_removed', +}) + + +def resolve_mcp_config( + global_home: str, + work_dir: Optional[str], + explicit_file: Optional[str] = None, +) -> Optional[Dict[str, Any]]: + """Build a ``{'mcpServers': {...}}`` dict for the agent's ``mcp_config`` + kwarg from the managed mcp.json files (+ optional explicit file), or None. + """ + servers: Dict[str, Any] = {} + try: + from ms_agent.config import MCPConfigManager + mm = MCPConfigManager(global_root=global_home, project_root=work_dir) + # 'merged' requires a project_root; fall back to 'global' without one. + scope = 'merged' if work_dir else 'global' + for name, entry in (mm.list(scope) or {}).items(): + entry = dict(entry) + if entry.get('enabled', True) is False: + continue # UI-disabled → do not connect + servers[name] = { + k: v for k, v in entry.items() if k not in _MCP_META + } + except Exception: + pass + # Explicit --mcp-server-file wins last (same-name replace). + if explicit_file and os.path.isfile(explicit_file): + try: + with open(explicit_file, 'r', encoding='utf-8') as f: + data = json.load(f) + src = data.get('mcpServers', data) if isinstance(data, dict) else {} + if isinstance(src, dict): + servers.update(src) + except Exception: + pass + return {'mcpServers': servers} if servers else None + + +def merge_skills_into_config(config, global_home: str, work_dir: Optional[str]): + """Append managed skill sources + disabled from skills.json into + ``config.skills`` (in place, returns config). Catalog dedups by skill_id.""" + try: + from ms_agent.config.skills_manager import SkillsConfigManager + from ms_agent.skill.sources import parse_skill_source + merged = SkillsConfigManager(global_dir=global_home).load_merged( + work_dir) + except Exception: + return config + src_strings = merged.get('sources') or [] + disabled = merged.get('disabled') or [] + if not src_strings and not disabled: + return config + + new_sources = [] + for s in src_strings: + try: + src = parse_skill_source(str(s)) + entry = {'type': src.type.value} + for k in ('path', 'repo_id', 'url', 'revision', 'subdir'): + v = getattr(src, k, None) + if v: + entry[k] = v + new_sources.append(entry) + except Exception: + continue + + skills = getattr(config, 'skills', None) + existing_sources = [] + existing_disabled = [] + if skills is not None: + raw = getattr(skills, 'sources', None) + if raw: + existing_sources = OmegaConf.to_container(raw, resolve=True) or [] + existing_disabled = list(getattr(skills, 'disabled', []) or []) + + combined_sources = list(existing_sources) + new_sources + combined_disabled = list( + dict.fromkeys(existing_disabled + list(disabled))) + if combined_sources: + OmegaConf.update(config, 'skills.sources', combined_sources, + merge=False) + if combined_disabled: + OmegaConf.update(config, 'skills.disabled', combined_disabled, + merge=False) + return config diff --git a/ms_agent/tui/permission.py b/ms_agent/tui/permission.py new file mode 100644 index 000000000..0e7a18090 --- /dev/null +++ b/ms_agent/tui/permission.py @@ -0,0 +1,95 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Permission handler for the TUI — an inline arrow-key confirmation menu. + +Matches the pattern used by Claude Code / Qoder / hermes: a compact header for +the tool call, then a selectable menu (``❯`` cursor, ↑/↓ + number keys, Enter) +instead of a "type a letter" prompt. The enforcer serializes asks and this runs +on the main event loop, so a prompt_toolkit menu composes without terminal +contention; a non-TTY fallback keeps it scriptable. +""" +from __future__ import annotations + +import asyncio +import json +from typing import Any, Optional + +from rich.console import Console + +from ms_agent.tui.select import select_async +from ms_agent.tui.theme import DEFAULT_THEME, Theme + +# Menu rows → PermissionAction. Order is the on-screen order. +_ALLOW_ONCE, _ALLOW_SESSION, _ALLOW_ALWAYS, _EDIT, _DENY = range(5) + + +class TUIPermissionHandler: + def __init__(self, console: Optional[Console] = None, io: Any = None, + theme: Theme = DEFAULT_THEME) -> None: + self._console = console or Console() + self._theme = theme + + async def ask(self, tool_name, tool_args, context, suggestions=None): + from ms_agent.permission.handler import (PermissionAction, + PermissionResponse) + suggestion = suggestions[0] if suggestions else tool_name + + # No persistent box: the tool line ("• Write path") already printed by + # the renderer is the context. The header below renders *inside* the + # transient menu and is erased with it, so once decided only the tool + # line + its result remain (Claude Code / Qoder style). + header = f'⚠ allow this tool call? {tool_name}' + args_preview = self._format_args(tool_args) + if args_preview: + header += '\n' + args_preview + if context: + header += '\n' + str(context) + + options = [ + 'Allow once', + 'Allow for this session', + f'Always allow [{suggestion}]', + 'Edit arguments', + 'Deny', + ] + idx = await select_async(options, default=_ALLOW_ONCE, header=header) + + # ── map selection → response (no persistent echo; the tool result line + # that follows is the trace — a denial shows "└ Tool call denied …") ── + if idx is None or idx == _DENY: + return PermissionResponse(action=PermissionAction.DENY) + + if idx == _ALLOW_SESSION: + return PermissionResponse(action=PermissionAction.ALLOW_SESSION, + pattern=suggestion) + if idx == _ALLOW_ALWAYS: + return PermissionResponse(action=PermissionAction.ALLOW_ALWAYS, + pattern=suggestion) + if idx == _EDIT: + raw = (await self._read_line('new args (JSON): ')).strip() + try: + new_args = json.loads(raw) + except (json.JSONDecodeError, ValueError): + self._console.print('[red]invalid JSON — denying[/]') + return PermissionResponse(action=PermissionAction.DENY) + return PermissionResponse(action=PermissionAction.MODIFY, + updated_args=new_args) + return PermissionResponse(action=PermissionAction.ALLOW_ONCE) + + def _format_args(self, tool_args) -> str: + try: + s = json.dumps(tool_args, ensure_ascii=False, indent=2) + except (TypeError, ValueError): + s = str(tool_args) + lines = s.splitlines() + if len(lines) > 8: + s = '\n'.join(lines[:8]) + '\n …' + elif len(s) > 400: + s = s[:400] + '…' + return s + + async def _read_line(self, prompt: str) -> str: + loop = asyncio.get_running_loop() + try: + return await loop.run_in_executor(None, input, prompt) + except (EOFError, KeyboardInterrupt): + return '' diff --git a/ms_agent/tui/renderer.py b/ms_agent/tui/renderer.py new file mode 100644 index 000000000..00ff12d6d --- /dev/null +++ b/ms_agent/tui/renderer.py @@ -0,0 +1,241 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Rich renderer: turns the agent's structured :class:`AgentEvent` stream into +a scrolling terminal transcript. + +This is a pure consumer of the ``ms_agent.ui`` event contract — it knows +nothing about the agent internals, only how to *draw* each semantic event. The +same event stream a WebUI backend forwards to a browser is what this renders to +a terminal, which is why the TUI validates that contract. + +Dispatch is open/closed: ``emit`` looks up ``_on_``; adding a new +event only needs a new handler method, never a change here. +""" +from __future__ import annotations + +import json +import re +import time +from typing import Optional + +from rich.console import Console +from rich.markdown import Markdown +from rich.markup import escape +from rich.panel import Panel +from rich.syntax import Syntax +from rich.text import Text + +from ms_agent.tui.state import TuiState +from ms_agent.tui.theme import DEFAULT_THEME, Theme +from ms_agent.tui.tool_view import tool_header, tool_summary +from ms_agent.ui.events import AgentEvent + +_KEY_LINE = re.compile(r'^\s*[\w.\-]+:(\s|$)') + + +def _looks_texty(s: str) -> bool: + t = s.lstrip() + return not (t.startswith('{') or t.startswith('[')) + + +def _guess_syntax(text: str) -> Optional[str]: + """Best-effort lexer for command output so /config etc. render nicely.""" + t = text.strip() + if not t: + return None + if t[0] in '{[': + try: + json.loads(t) + return 'json' + except (json.JSONDecodeError, ValueError): + pass + if sum(1 for ln in text.splitlines() if _KEY_LINE.match(ln)) >= 3: + return 'yaml' + return None + + +class RichEventSink: + """An :class:`~ms_agent.ui.events.AgentEventSink` that renders with rich.""" + + def __init__(self, console: Console, state: TuiState, + theme: Theme = DEFAULT_THEME) -> None: + self.console = console + self.state = state + self.theme = theme + self._live = None + self._content_buf = '' + self._label_shown = False + self._reasoning_active = False + self._tool_started: dict = {} # call_id -> monotonic_start + # Kind of the last block rendered ('tool' | 'content' | None), used to + # add a blank line between sections (tools ↔ assistant) for breathing + # room without spacing tightly-grouped tool lines apart. + self._last_kind: Optional[str] = None + + # ── sink protocol ────────────────────────────────────────────────────── + + def emit(self, event: AgentEvent) -> None: + handler = getattr(self, f'_on_{event.type}', None) + if handler is not None: + handler(event) + + def finalize(self) -> None: + """Tear down any in-flight Live region (call on error / turn abort).""" + if self._live is not None: + try: + self._live.stop() + except Exception: + pass + self._live = None + self._content_buf = '' + self._label_shown = False + + # ── assistant content (streamed) ─────────────────────────────────────── + + def _on_content_delta(self, ev) -> None: + if not self._label_shown: + # Always a blank line before the assistant reply — a clear gap after + # the user prompt (or the preceding tools/thinking). + self.console.print() + self.console.print( + f'[{self.theme.assistant}]{self.theme.assistant_symbol} ' + f'Assistant[/]') + self._label_shown = True + self._last_kind = 'content' + self._content_buf += ev.text + if self.console.is_terminal: + if self._live is None: + from rich.live import Live + self._live = Live(console=self.console, refresh_per_second=12, + vertical_overflow='visible') + self._live.start() + # Stream as plain Text (tolerant of half-formed markdown); the + # final ContentEnd reflows once into formatted Markdown. + self._live.update(Text(self._content_buf)) + else: + # Non-terminal (piped / CI): write raw for clean, scriptable output. + self.console.file.write(ev.text) + self.console.file.flush() + + def _on_content_end(self, ev) -> None: + buf = self._content_buf + if self._live is not None: + renderable = Markdown(buf) if buf.strip() else Text('') + self._live.update(renderable) + self._live.stop() + self._live = None + elif not self.console.is_terminal: + self.console.file.write('\n') + self.console.file.flush() + self._content_buf = '' + self._label_shown = False + self._last_kind = None # next turn starts fresh (input divider spaces it) + + # ── reasoning (collapsed dim block) ───────────────────────────────────── + + def _on_reasoning_started(self, ev) -> None: + # A gap + header, then the reasoning streams dim below it (always spaced + # off whatever preceded — a tool result, the prompt, etc.). + self.console.print() + self.console.print(f'[{self.theme.reasoning}]✻ Thinking[/]') + self._reasoning_active = True + self._last_kind = 'content' + + def _on_reasoning_delta(self, ev) -> None: + if self._reasoning_active and ev.text: + # Stream tokens as they arrive (dim), same as assistant content. + self.console.print( + Text(ev.text, style=self.theme.reasoning), end='') + + def _on_reasoning_ended(self, ev) -> None: + if self._reasoning_active: + self.console.print() # close the streamed thinking block + self._reasoning_active = False + + # ── tool calls ───────────────────────────────────────────────────────── + + def _on_tool_call_started(self, ev) -> None: + # Compact action line (Codex style): "• Write path/to/file". + if self._last_kind != 'tool': + # Blank before a new tool group (turn start or after text), but keep + # consecutive tool lines tight together. + self.console.print() + self._tool_started[ev.call_id] = time.monotonic() + header = escape(tool_header(ev.name, ev.arguments)) + self.console.print(f'[{self.theme.tool_bullet}]•[/] {header}') + self._last_kind = 'tool' + + def _on_tool_call_completed(self, ev) -> None: + # Indented one-line summary: " └ 42 lines · 1.2s". + start = self._tool_started.pop(ev.call_id, None) + dur = f' · {time.monotonic() - start:.1f}s' if start else '' + if ev.error: + summary = escape(tool_summary(ev.result, ev.error)) + self.console.print( + f' [{self.theme.tool_error_border}]└[/] ' + f'[{self.theme.tool_error_border}]{summary}[/][dim]{dur}[/]') + else: + summary = escape(tool_summary(ev.result)) + self.console.print(f' [dim]└ {summary}{dur}[/]') + + # ── plan / notices / context / errors ────────────────────────────────── + + def _on_plan_updated(self, ev) -> None: + if not ev.entries: + return + mark = {'completed': '[green]✔[/]', 'in_progress': '[yellow]▸[/]'} + lines = [] + for e in ev.entries: + # entries may be PlanEntry objects (live) or dicts (deserialized). + status = (e.get('status', 'pending') if isinstance(e, dict) + else getattr(e, 'status', 'pending')) + content = (e.get('content', '') if isinstance(e, dict) + else getattr(e, 'content', '')) + lines.append(f'{mark.get(status, "○")} {content}') + self.console.print( + Panel('\n'.join(lines), title='plan', border_style='blue', + expand=False)) + + def _on_context_compacted(self, ev) -> None: + detail = '' + if ev.before_tokens and ev.after_tokens: + detail = f' {ev.before_tokens}→{ev.after_tokens} tok' + self.console.print( + f'[{self.theme.notice_info}]· context compacted{detail} ·[/]') + + def _on_notice(self, ev) -> None: + if ev.level == 'info': + # Command output (/help, /model, /config …): a subtle panel; when + # the content is structured (YAML/JSON, e.g. /config) syntax- + # highlight it, else render as markup-safe text. + lexer = _guess_syntax(ev.text) + if lexer: + body = Syntax(ev.text, lexer, theme='ansi_dark', + word_wrap=True, background_color='default') + else: + body = Text(ev.text) + self.console.print( + Panel(body, border_style='dim', expand=False)) + return + style = {'success': self.theme.notice_success, + 'warning': self.theme.notice_warning}.get( + ev.level, self.theme.notice_info) + self.console.print(f'[{style}]{ev.text}[/]') + + def _on_error(self, ev) -> None: + self.finalize() + self.console.print( + Panel(f'[bold]{ev.message}[/]', title='error', + border_style=self.theme.error_border, expand=False)) + + def _on_turn_completed(self, ev) -> None: + if ev.usage is not None: + self.state.total_prompt_tokens = ev.usage.total_prompt_tokens + self.state.total_completion_tokens = ev.usage.total_completion_tokens + + # ── display helpers (called directly by the app, not via events) ──────── + + def rule(self, text: str, style: Optional[str] = None) -> None: + self.console.rule(f'[{style or self.theme.rule}]{text}[/]') + + def notice(self, text: str, level: str = 'info') -> None: + self._on_notice(type('N', (), {'text': text, 'level': level})()) diff --git a/ms_agent/tui/select.py b/ms_agent/tui/select.py new file mode 100644 index 000000000..48327c753 --- /dev/null +++ b/ms_agent/tui/select.py @@ -0,0 +1,126 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Inline arrow-key selection menu (prompt_toolkit). + +A small, reusable single-choice picker rendered *in the scroll flow* (not a +full-screen dialog) — the pattern Claude Code / Qoder / hermes use for +permission prompts and model pickers: a ``❯`` cursor on the highlighted row, +↑/↓ (or Ctrl-P/N) to move, number keys to jump, Enter to confirm, Esc/Ctrl-C +to cancel. The menu erases itself on exit so only the outcome remains. + +Runs on the current event loop via ``run_async`` (the agent's permission ask +executes in the main loop, so this composes cleanly). Callers must guard for a +TTY; there is a plain fallback for pipes/CI. +""" +from __future__ import annotations + +import sys +from typing import List, Optional, Sequence + + +async def select_async(options: Sequence[str], *, default: int = 0, + header: Optional[str] = None) -> Optional[int]: + """Show an inline menu; return the chosen index, or None if cancelled. + + ``header`` (optional, may be multi-line) renders above the options *inside* + the transient menu — its first line bold, the rest dim — so context (e.g. a + tool name + args) shows during the decision and is erased with the menu, + leaving nothing behind. + + Falls back to a single blocking line read when stdin is not a TTY (accepts + a 1-based number). + """ + if not options: + return None + if not sys.stdin.isatty(): + return await _fallback_numeric(len(options)) + return await _menu_async(options, default, header) + + +async def _menu_async(options: Sequence[str], default: int, + header: Optional[str] = None) -> Optional[int]: + """The prompt_toolkit menu itself (no TTY guard, so tests can drive it via + a pipe input + AppSession).""" + from prompt_toolkit import Application + from prompt_toolkit.key_binding import KeyBindings + from prompt_toolkit.layout import HSplit, Layout, Window + from prompt_toolkit.layout.controls import FormattedTextControl + from prompt_toolkit.styles import Style + + sel = [max(0, min(default, len(options) - 1))] + kb = KeyBindings() + + @kb.add('up') + @kb.add('c-p') + def _up(event) -> None: + sel[0] = (sel[0] - 1) % len(options) + + @kb.add('down') + @kb.add('c-n') + @kb.add('tab') + def _down(event) -> None: + sel[0] = (sel[0] + 1) % len(options) + + @kb.add('enter') + def _accept(event) -> None: + event.app.exit(result=sel[0]) + + @kb.add('escape') + @kb.add('c-c') + def _cancel(event) -> None: + event.app.exit(result=None) + + for _i in range(min(len(options), 9)): + @kb.add(str(_i + 1)) + def _pick(event, i=_i) -> None: + event.app.exit(result=i) + + header_lines = header.splitlines() if header else [] + + def _render(): + frags = [] + for j, hl in enumerate(header_lines): + frags.append(('class:head' if j == 0 else 'class:headdim', + hl + '\n')) + for i, label in enumerate(options): + if i == sel[0]: + frags.append(('class:sel', f'❯ {i + 1}. {label}\n')) + else: + frags.append(('class:opt', f' {i + 1}. {label}\n')) + frags.append(('class:hint', '↑/↓ move · enter select · esc cancel')) + return frags + + control = FormattedTextControl(_render, focusable=True, show_cursor=False) + style = Style.from_dict({ + 'sel': 'bold ansicyan', + 'opt': '', + 'head': 'bold ansiyellow', + 'headdim': 'ansibrightblack', + 'hint': 'italic ansibrightblack', + }) + app = Application( + layout=Layout(HSplit([Window( + control, height=len(options) + 1 + len(header_lines))])), + key_bindings=kb, + style=style, + full_screen=False, + erase_when_done=True, + mouse_support=False, + ) + return await app.run_async() + + +async def _fallback_numeric(n: int) -> Optional[int]: + """Non-TTY fallback: read a 1-based number (or legacy letter) from stdin.""" + import asyncio + loop = asyncio.get_running_loop() + try: + raw = (await loop.run_in_executor(None, input, 'choice: ')).strip() + except (EOFError, KeyboardInterrupt): + return None + if raw.isdigit() and 1 <= int(raw) <= n: + return int(raw) - 1 + return _LEGACY_LETTERS.get(raw.lower()) + + +# Back-compat for scripted input that still sends y/s/a/e/n. +_LEGACY_LETTERS = {'y': 0, 's': 1, 'a': 2, 'e': 3, 'n': 4} diff --git a/ms_agent/tui/state.py b/ms_agent/tui/state.py new file mode 100644 index 000000000..953330ef0 --- /dev/null +++ b/ms_agent/tui/state.py @@ -0,0 +1,25 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Shared, mutable TUI state. + +A tiny value object the renderer writes (token usage, busy flag) and the input +bar reads (to draw the persistent bottom status line). Keeping it in one place +avoids threading half a dozen fields through both components. +""" +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass +class TuiState: + model: str = '' + perm: str = '' + work_dir: str = '' + session_name: str = '' + total_prompt_tokens: int = 0 + total_completion_tokens: int = 0 + busy: bool = False + + @property + def total_tokens(self) -> int: + return self.total_prompt_tokens + self.total_completion_tokens diff --git a/ms_agent/tui/theme.py b/ms_agent/tui/theme.py new file mode 100644 index 000000000..7090596af --- /dev/null +++ b/ms_agent/tui/theme.py @@ -0,0 +1,43 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Centralized TUI styling — one place to tune colors, so nothing is hardcoded +across the renderer / input / permission modules. Values are ``rich`` style +strings; keep them theme-neutral (readable on both light and dark terminals). +""" +from __future__ import annotations + +from dataclasses import dataclass, field + + +@dataclass(frozen=True) +class Theme: + user: str = 'bold cyan' + assistant: str = 'bold green' + assistant_label: str = 'green' + reasoning: str = 'dim' + tool_bullet: str = 'cyan' + tool_border: str = 'yellow' + tool_name: str = 'bold yellow' + tool_result_border: str = 'green' + tool_error_border: str = 'red' + error_border: str = 'red' + notice_info: str = 'dim' + notice_success: str = 'green' + notice_warning: str = 'yellow' + permission_border: str = 'magenta' + banner_border: str = 'cyan' + banner_label: str = 'bold cyan' + rule: str = 'cyan' + session_marker: str = 'bold green' + status_bar: str = 'dim' + prompt: str = 'bold cyan' + hint: str = 'dim' + + # symbols (kept here so the visual language is consistent + swappable) + prompt_symbol: str = '❯' + user_symbol: str = '❯' + assistant_symbol: str = '●' + tool_symbol: str = '▸' + result_symbol: str = '←' + + +DEFAULT_THEME = Theme() diff --git a/ms_agent/tui/tool_view.py b/ms_agent/tui/tool_view.py new file mode 100644 index 000000000..d9c6a76b7 --- /dev/null +++ b/ms_agent/tui/tool_view.py @@ -0,0 +1,87 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Compact tool-call presentation (Codex / hermes style). + +Turns a ``server---tool`` name + args into a one-line action header +(``Write path/to/file``, ``Run git push``, ``Search "modelscope"``) and a +tool result into a one-line summary (``42 lines``, ``(no output)``, +``error: …``). Pattern-based, not a per-tool hardcode: a small verb map for +common tools + a salient-argument heuristic, with a readable generic fallback. +""" +from __future__ import annotations + +import json +from typing import Any + +TOOL_SPLITER = '---' + +# action word (part after '---') → display verb +_VERBS = { + 'write_file': 'Write', 'edit_file': 'Edit', 'read_file': 'Read', + 'append_file': 'Append', 'delete_file': 'Delete', 'move_file': 'Move', + 'shell_executor': 'Run', 'execute': 'Run', 'run_command': 'Run', + 'exa_search': 'Search', 'web_search': 'Search', 'search': 'Search', + 'skill_view': 'View skill', 'skill_manage': 'Skill', + 'skills_list': 'List skills', 'glob': 'Find', 'grep': 'Search', + 'list_dir': 'List', 'read_dir': 'List', +} +# arg keys probed, in priority order, for the salient value to show +_ARG_KEYS = ('path', 'file_path', 'filename', 'query', 'q', 'command', 'cmd', + 'skill_id', 'url', 'pattern', 'directory', 'dir', 'name') + + +def _short(s: str, n: int = 72) -> str: + s = ' '.join(str(s).split()) # collapse whitespace/newlines + return s if len(s) <= n else s[:n] + '…' + + +def _coerce_args(args: Any) -> Any: + if isinstance(args, str): + try: + return json.loads(args) + except (json.JSONDecodeError, ValueError): + return args + return args + + +def _salient_arg(args: Any) -> str: + args = _coerce_args(args) + if isinstance(args, str): + return _short(args) + if isinstance(args, dict): + for k in _ARG_KEYS: + v = args.get(k) + if v: + return _short(str(v)) + for v in args.values(): # fall back to first scalar + if isinstance(v, (str, int, float)) and str(v).strip(): + return _short(str(v)) + return '' + + +def tool_header(name: str, args: Any) -> str: + """One-line action header, e.g. ``Write SKILL.md`` / ``Run git push``.""" + short = name.split(TOOL_SPLITER)[-1] if TOOL_SPLITER in name else name + arg = _salient_arg(args) + verb = _VERBS.get(short) + if verb: + return f'{verb} {arg}'.rstrip() + label = short.replace('_', ' ') + return f'{label} {arg}'.rstrip() if arg else label + + +def tool_summary(result: str, error: str | None = None) -> str: + """One-line result summary, e.g. ``42 lines`` / ``(no output)``.""" + if error: + return f'error: {_short(error)}' + r = (result or '').strip() + if not r: + return '(no output)' + lines = r.splitlines() + first = lines[0].strip() + if len(lines) == 1: + return _short(first) + # Lead with a line count when the first line is a trivial opener (JSON/ + # array/fence), which would otherwise render a meaningless "{ (+N lines)". + if len(first) <= 2 or first in ('{', '[', '```', '---'): + return f'{len(lines)} lines' + return f'{_short(first, 56)} (+{len(lines) - 1} lines)' diff --git a/ms_agent/ui/__init__.py b/ms_agent/ui/__init__.py new file mode 100644 index 000000000..164a27e88 --- /dev/null +++ b/ms_agent/ui/__init__.py @@ -0,0 +1,65 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""UI-agnostic contract layer. + +The stable abstraction every front-end depends on (TUI now, WebUI backend and +ACP later). Agent core depends on these protocols, not on any concrete UI — +so the same structured event + async-input contracts validate the WebUI +interface design. Pure stdlib; importing this package pulls in no rich / +prompt_toolkit / omegaconf. +""" +from ms_agent.ui.events import ( + AgentEvent, + AgentEventSink, + ContentDelta, + ContentEnd, + ContextCompacted, + ErrorRaised, + Notice, + PermissionRequested, + PermissionResolved, + PlanEntry, + PlanUpdated, + ReasoningDelta, + ReasoningEnded, + ReasoningStarted, + RecordingSink, + JsonlEventSink, + TeeEventSink, + ToolCallCompleted, + ToolCallStarted, + TurnCompleted, + TurnStarted, + UsageInfo, + UserMessage, +) +from ms_agent.ui.input import InputSource, StdinInputSource + +__all__ = [ + # events + 'AgentEvent', + 'AgentEventSink', + 'TurnStarted', + 'UserMessage', + 'TurnCompleted', + 'ContentDelta', + 'ContentEnd', + 'ReasoningStarted', + 'ReasoningDelta', + 'ReasoningEnded', + 'ToolCallStarted', + 'ToolCallCompleted', + 'PlanEntry', + 'PlanUpdated', + 'PermissionRequested', + 'PermissionResolved', + 'ContextCompacted', + 'Notice', + 'ErrorRaised', + 'UsageInfo', + 'RecordingSink', + 'TeeEventSink', + 'JsonlEventSink', + # input + 'InputSource', + 'StdinInputSource', +] diff --git a/ms_agent/ui/events.py b/ms_agent/ui/events.py new file mode 100644 index 000000000..8ab1f60c8 --- /dev/null +++ b/ms_agent/ui/events.py @@ -0,0 +1,287 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""UI-agnostic structured event model for agent output. + +This module is the **contract** every front-end (TUI, future WebUI backend, +ACP translator) consumes. The agent emits :class:`AgentEvent` instances to an +:class:`AgentEventSink`; each event is an immutable, trivially-serializable +dataclass whose ``to_dict()`` *is* the wire format (WebSocket / SSE payload). + +Design rules (keep this module dependency-free — pure stdlib only): + +* **Semantic, not pixels** — events ship meaning (``ToolCallStarted``), never + rendered text. A renderer decides how to draw them. +* **Open/closed** — add a new event by adding a frozen dataclass; the + ``emit(event)`` protocol never changes. +* **Single source of truth** — the ``EVENT_TYPE`` string is the discriminator + a WebUI front-end mirrors; do not rename lightly. + +The vocabulary maps 1:1 onto ACP ``session_update`` types (see +``ms_agent/acp/translator.py``) and covers the two gaps ACP leaves +(``UserMessage`` echo, command catalog), so the same stream can drive ACP, +a WebUI, and the TUI without a second protocol. +""" +from __future__ import annotations + +from dataclasses import asdict, dataclass, field +from typing import Any, ClassVar, List, Optional, Protocol, runtime_checkable + +# ── value objects ───────────────────────────────────────────────────────── + + +@dataclass(frozen=True) +class UsageInfo: + """Token accounting carried on turn completion (drives the status bar).""" + prompt_tokens: int = 0 + completion_tokens: int = 0 + reasoning_tokens: int = 0 + total_prompt_tokens: int = 0 + total_completion_tokens: int = 0 + + +@dataclass(frozen=True) +class PlanEntry: + """One item of a plan/todo list (from the todo / split_task tools).""" + content: str + status: str = 'pending' # pending | in_progress | completed + + +# ── event base ──────────────────────────────────────────────────────────── + + +@dataclass(frozen=True) +class AgentEvent: + """Base class for all agent events. Immutable and serializable. + + ``EVENT_TYPE`` is a class-level discriminator (not a dataclass field), so + it is excluded from ``asdict`` but injected by :meth:`to_dict`. + """ + EVENT_TYPE: ClassVar[str] = 'agent_event' + + @property + def type(self) -> str: + return self.EVENT_TYPE + + def to_dict(self) -> dict: + """Return the serializable wire form: ``{'type': ..., **fields}``.""" + return {'type': self.EVENT_TYPE, **asdict(self)} + + +# ── turn lifecycle ──────────────────────────────────────────────────────── + + +@dataclass(frozen=True) +class TurnStarted(AgentEvent): + """A user turn began (one user input → one or more assistant steps).""" + EVENT_TYPE: ClassVar[str] = 'turn_started' + turn_id: str = '' + source: str = 'tui' # cli | tui | webui + + +@dataclass(frozen=True) +class UserMessage(AgentEvent): + """Echo of the user's submitted text (ACP's missing user_message_chunk).""" + EVENT_TYPE: ClassVar[str] = 'user_message' + turn_id: str = '' + text: str = '' + + +@dataclass(frozen=True) +class TurnCompleted(AgentEvent): + """The assistant finished responding to the current turn.""" + EVENT_TYPE: ClassVar[str] = 'turn_completed' + turn_id: str = '' + usage: Optional[UsageInfo] = None + + +# ── assistant output ────────────────────────────────────────────────────── + + +@dataclass(frozen=True) +class ContentDelta(AgentEvent): + """An incremental chunk of assistant message content.""" + EVENT_TYPE: ClassVar[str] = 'content_delta' + text: str = '' + + +@dataclass(frozen=True) +class ContentEnd(AgentEvent): + """The assistant content stream for the current message finished. + + A renderer uses this to finalize a streaming bubble (e.g. re-render the + accumulated text as Markdown). + """ + EVENT_TYPE: ClassVar[str] = 'content_end' + + +@dataclass(frozen=True) +class ReasoningStarted(AgentEvent): + """The model began emitting reasoning / thinking tokens.""" + EVENT_TYPE: ClassVar[str] = 'reasoning_started' + + +@dataclass(frozen=True) +class ReasoningDelta(AgentEvent): + """An incremental chunk of reasoning / thinking content.""" + EVENT_TYPE: ClassVar[str] = 'reasoning_delta' + text: str = '' + + +@dataclass(frozen=True) +class ReasoningEnded(AgentEvent): + """The reasoning / thinking stream finished.""" + EVENT_TYPE: ClassVar[str] = 'reasoning_ended' + + +# ── tools ───────────────────────────────────────────────────────────────── + + +@dataclass(frozen=True) +class ToolCallStarted(AgentEvent): + """A tool call is about to execute.""" + EVENT_TYPE: ClassVar[str] = 'tool_call_started' + call_id: str = '' + name: str = '' + arguments: Any = None # dict or raw JSON string + + +@dataclass(frozen=True) +class ToolCallCompleted(AgentEvent): + """A tool call finished (successfully or with an error).""" + EVENT_TYPE: ClassVar[str] = 'tool_call_completed' + call_id: str = '' + name: str = '' + result: str = '' + error: Optional[str] = None + duration_s: Optional[float] = None + + +# ── plan / todo ─────────────────────────────────────────────────────────── + + +@dataclass(frozen=True) +class PlanUpdated(AgentEvent): + """The agent's plan / todo list changed.""" + EVENT_TYPE: ClassVar[str] = 'plan_updated' + entries: List[PlanEntry] = field(default_factory=list) + + +# ── permission (human-in-the-loop) ──────────────────────────────────────── + + +@dataclass(frozen=True) +class PermissionRequested(AgentEvent): + """A tool needs user authorization (mirrors WebPermissionHandler emit).""" + EVENT_TYPE: ClassVar[str] = 'permission_requested' + request_id: str = '' + tool_name: str = '' + tool_args: Any = None + context: str = '' + options: List[str] = field(default_factory=list) + + +@dataclass(frozen=True) +class PermissionResolved(AgentEvent): + """A pending permission request was answered.""" + EVENT_TYPE: ClassVar[str] = 'permission_resolved' + request_id: str = '' + action: str = '' # e.g. allow_once | deny | allow_always + + +# ── context / system notices ────────────────────────────────────────────── + + +@dataclass(frozen=True) +class ContextCompacted(AgentEvent): + """Non-destructive context compaction ran (long-conversation upkeep).""" + EVENT_TYPE: ClassVar[str] = 'context_compacted' + before_tokens: int = 0 + after_tokens: int = 0 + + +@dataclass(frozen=True) +class Notice(AgentEvent): + """A user-facing informational notice.""" + EVENT_TYPE: ClassVar[str] = 'notice' + level: str = 'info' # info | success | warning + text: str = '' + + +@dataclass(frozen=True) +class ErrorRaised(AgentEvent): + """An error surfaced during the run.""" + EVENT_TYPE: ClassVar[str] = 'error' + message: str = '' + recoverable: bool = True + + +# ── sink protocol + reference implementations ───────────────────────────── + + +@runtime_checkable +class AgentEventSink(Protocol): + """The seam the agent emits to. A front-end implements ``emit``. + + Implementations must be cheap and non-blocking — ``emit`` is called on the + hot path of token streaming. Rendering/formatting is the sink's business. + """ + + def emit(self, event: AgentEvent) -> None: + ... + + +class RecordingSink: + """Collects every emitted event. Used by tests and the ``--emit-events`` + debug dump; also handy as a base for buffering front-ends.""" + + def __init__(self) -> None: + self.events: List[AgentEvent] = [] + + def emit(self, event: AgentEvent) -> None: + self.events.append(event) + + def types(self) -> List[str]: + return [e.type for e in self.events] + + def of_type(self, event_type: str) -> List[AgentEvent]: + return [e for e in self.events if e.type == event_type] + + def text(self) -> str: + """Concatenated assistant content — a quick assertion helper.""" + return ''.join(e.text for e in self.events + if isinstance(e, ContentDelta)) + + +class TeeEventSink: + """Fans an event out to several sinks (e.g. render + record to disk).""" + + def __init__(self, *sinks: AgentEventSink) -> None: + self._sinks = [s for s in sinks if s is not None] + + def emit(self, event: AgentEvent) -> None: + for sink in self._sinks: + sink.emit(event) + + +class JsonlEventSink: + """Writes each event as one JSON line — a dump of the exact wire payloads a + WebUI backend would forward. Use it to inspect and validate that the event + contract carries everything a front-end needs. + """ + + def __init__(self, path) -> None: + import json + self._json = json + self._own = isinstance(path, str) + self._f = open(path, 'a', encoding='utf-8') if self._own else path + + def emit(self, event: AgentEvent) -> None: + self._f.write( + self._json.dumps(event.to_dict(), ensure_ascii=False) + '\n') + self._f.flush() + + def close(self) -> None: + if self._own: + try: + self._f.close() + except Exception: + pass diff --git a/ms_agent/ui/input.py b/ms_agent/ui/input.py new file mode 100644 index 000000000..a44b0c653 --- /dev/null +++ b/ms_agent/ui/input.py @@ -0,0 +1,48 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""UI-agnostic asynchronous input contract. + +The agent's interactive loop obtains the next user prompt through an +:class:`InputSource`. Making ``read_prompt`` *awaitable* is what lets a rich +TUI (and a future WebUI backend) drive the native ``run_loop`` without the +input read blocking the event loop or fighting a streaming renderer for the +terminal — the enabling seam for "route A" (agent owns one lifecycle). + +* **TUI** implements it over ``prompt_toolkit.PromptSession.prompt_async``. +* **WebUI backend** implements it as ``await queue.get()`` (input pushed by a + WebSocket handler). +* **CLI** default is :class:`StdinInputSource` — a blocking ``input()`` run in + a thread executor, so behavior is identical to bare ``input()`` while still + cooperating with asyncio. + +Pure stdlib; no front-end dependencies. +""" +from __future__ import annotations + +import asyncio +from typing import Protocol, runtime_checkable + + +@runtime_checkable +class InputSource(Protocol): + """Async source of user prompts. + + ``read_prompt`` should raise ``EOFError`` (end of input) or + ``KeyboardInterrupt`` (user abort) to signal that the session should quit; + callers (``InteractiveSession``) translate those into a quit turn. + """ + + async def read_prompt(self, prompt: str = '>>> ') -> str: + ... + + +class StdinInputSource: + """Default input source: blocking ``input()`` off the event loop. + + Functionally identical to bare ``input()`` (still blocks for a line), but + runs in the default executor so a single asyncio loop stays responsive to + other tasks (e.g. a concurrent ``/stop`` reader) while awaiting input. + """ + + async def read_prompt(self, prompt: str = '>>> ') -> str: + loop = asyncio.get_running_loop() + return await loop.run_in_executor(None, input, prompt) diff --git a/ms_agent/utils/artifact_manager.py b/ms_agent/utils/artifact_manager.py index 2d73457f0..35ddc74b0 100644 --- a/ms_agent/utils/artifact_manager.py +++ b/ms_agent/utils/artifact_manager.py @@ -19,7 +19,7 @@ def __init__( max_combined_bytes: int = 256 * 1024, preview_head_chars: int = 4000, preview_tail_chars: int = 2000, - artifact_subdir: str = '.ms_agent_artifacts', + artifact_subdir: str = '.ms_agent/artifacts', ) -> None: self._root = Path(output_dir).expanduser().resolve() self.max_combined_bytes = max_combined_bytes diff --git a/ms_agent/utils/constants.py b/ms_agent/utils/constants.py index cbd37e271..d502bd173 100644 --- a/ms_agent/utils/constants.py +++ b/ms_agent/utils/constants.py @@ -5,9 +5,11 @@ # When ``output_dir`` is omitted, ``resolve_workspace_root()`` uses cwd instead. DEFAULT_OUTPUT_DIR = './output' -DEFAULT_INDEX_DIR = '.index' +# Framework-internal subdirs, collapsed under /.ms_agent/ (see +# ms_agent.project.paths) so nothing scatters at the work-dir root. +DEFAULT_INDEX_DIR = '.ms_agent/index' -DEFAULT_LOCK_DIR = '.locks' +DEFAULT_LOCK_DIR = '.ms_agent/locks' # The key of user defined tools in the agent.yaml TOOL_PLUGIN_NAME = 'plugins' @@ -18,8 +20,10 @@ # Default agent code file DEFAULT_AGENT_FILE = 'agent.py' -# Default memory folder -DEFAULT_MEMORY_DIR = '.memory' +# History cache folder, collapsed under the framework-internal .ms_agent/ dir +# (was '.memory'). LEGACY_MEMORY_DIR is still read for back-compat. +DEFAULT_MEMORY_DIR = '.ms_agent/memory' +LEGACY_MEMORY_DIR = '.memory' # DEFAULT_WORKFLOW_YAML WORKFLOW_CONFIG_FILE = 'workflow.yaml' diff --git a/ms_agent/utils/logger.py b/ms_agent/utils/logger.py index 0ac9d48a8..aefe47750 100644 --- a/ms_agent/utils/logger.py +++ b/ms_agent/utils/logger.py @@ -30,14 +30,39 @@ def warning_once(self, msg, *args, **kwargs): self.warning(msg) +def _resolve_log_file(log_file: Optional[str]) -> Optional[str]: + """File logging is opt-in. Returns a path only when explicitly requested + (``log_file`` arg or ``MS_AGENT_LOG_FILE`` / ``LOG_FILE`` env); otherwise + ``None`` (console-only) so we never scatter ``ms_agent.log`` in the CWD. + + A truthy-flag env value (``1``/``true``) routes to the global + ``~/.ms_agent/logs/ms_agent.log``; an explicit path is used as-is. + """ + if log_file: + return log_file + env = os.environ.get('MS_AGENT_LOG_FILE') or os.environ.get('LOG_FILE') + if not env: + return None + if env.strip().lower() in ('1', 'true', 'yes', 'on'): + from ms_agent.project.paths import global_logs_dir + d = global_logs_dir() + try: + d.mkdir(parents=True, exist_ok=True) + return str(d / 'ms_agent.log') + except OSError: + # Never let logging setup crash the app; fall back to console-only. + return None + return env + + def get_logger(log_file: Optional[str] = None, log_level: Optional[int] = None, file_mode: str = 'w'): """ Get logging logger Args: - log_file: Log filename, if specified, file handler will be added to - logger. If None, defaults to 'ms_agent.log' in current working directory. + log_file: Log filename. File logging is opt-in: when omitted (and no + ``MS_AGENT_LOG_FILE``/``LOG_FILE`` env), logging is console-only. log_level: Logging level. file_mode: Specifies the mode to open the file, if filename is specified (if filemode is unspecified, it defaults to 'w'). @@ -46,9 +71,7 @@ def get_logger(log_file: Optional[str] = None, log_level = os.getenv('LOG_LEVEL', 'INFO').upper() log_level = getattr(logging, log_level, logging.INFO) - # Default log file path: current working directory - if log_file is None: - log_file = os.path.join(os.getcwd(), 'ms_agent.log') + log_file = _resolve_log_file(log_file) logger_name = __name__.split('.')[0] logger = logging.getLogger(logger_name) logger.propagate = False @@ -74,9 +97,9 @@ def get_logger(log_file: Optional[str] = None, stream_handler = logging.StreamHandler() handlers = [stream_handler] - # Always add file handler since log_file is set to default if None - file_handler = logging.FileHandler(log_file, file_mode) - handlers.append(file_handler) + # File handler only when opt-in (see _resolve_log_file); console otherwise. + if log_file: + handlers.append(logging.FileHandler(log_file, file_mode)) for handler in handlers: handler.setFormatter(logger_format) @@ -121,9 +144,9 @@ def add_file_handler_if_needed(logger, log_file, file_mode, log_level): is_worker0 = True if is_worker0: - # Default log file path if not specified + # File logging is opt-in; no CWD default. if log_file is None: - log_file = os.path.join(os.getcwd(), 'ms_agent.log') + return file_handler = logging.FileHandler(log_file, file_mode) file_handler.setFormatter(logger_format) file_handler.setLevel(log_level) diff --git a/ms_agent/utils/snapshot.py b/ms_agent/utils/snapshot.py index d7362405a..d731c20ec 100644 --- a/ms_agent/utils/snapshot.py +++ b/ms_agent/utils/snapshot.py @@ -17,7 +17,6 @@ logger = get_logger() -_SNAPSHOT_DIR_NAME = '.ms_agent_snapshots' _META_FILE = 'snapshot_meta.json' @@ -41,7 +40,9 @@ def _git(args: list[str], def _snapshot_git_dir(output_dir: str) -> str: - return os.path.join(output_dir, _SNAPSHOT_DIR_NAME) + # Collapsed under /.ms_agent/ (was /.ms_agent_snapshots). + from ms_agent.project.paths import snapshots_dir + return str(snapshots_dir(output_dir)) def _configure_snapshot_repo_for_automation(work_tree: str, @@ -81,7 +82,11 @@ def _ensure_repo(output_dir: str) -> str: os.makedirs(info_dir, exist_ok=True) exclude_file = os.path.join(info_dir, 'exclude') with open(exclude_file, 'a', encoding='utf-8') as f: - f.write(f'\n{_SNAPSHOT_DIR_NAME}/\n') + # Exclude only the snapshot git repo itself (must not self-track). + # The rest of .ms_agent/ (history cache, etc.) is still captured, + # preserving the pre-move behavior where /.memory was + # part of the snapshot. + f.write('\n.ms_agent/snapshots/\n') # Always (re)apply: repos created before this fix may still inherit hooks. _configure_snapshot_repo_for_automation(output_dir, git_dir) return git_dir diff --git a/ms_agent/utils/stats.py b/ms_agent/utils/stats.py index beeccfc1f..be49661a4 100644 --- a/ms_agent/utils/stats.py +++ b/ms_agent/utils/stats.py @@ -25,12 +25,13 @@ def _get_lock(path: str) -> asyncio.Lock: def get_stats_path(config: Any, default_filename: str = 'workflow_stats.json') -> str: stats_file = getattr(config, 'stats_file', None) - output_dir = getattr(config, 'output_dir', './output') + output_dir = getattr(config, 'output_dir', None) or '.' if stats_file: if os.path.isabs(stats_file): return stats_file return os.path.join(output_dir, stats_file) - return os.path.join(output_dir, default_filename) + from ms_agent.project.paths import stats_file as _stats_path + return str(_stats_path(output_dir, default_filename)) def summarize_usage(messages: Optional[Iterable[Message]]) -> Dict[str, int]: diff --git a/ms_agent/utils/stream_writer.py b/ms_agent/utils/stream_writer.py index cdeddec77..18f74ef8f 100644 --- a/ms_agent/utils/stream_writer.py +++ b/ms_agent/utils/stream_writer.py @@ -52,7 +52,7 @@ def __init__(self, output_dir: str, call_id: str, tool_name: str) -> None: self._agent_tag: Optional[str] = None self._file = None # opened lazily in on_start - subagents_dir = os.path.join(output_dir, 'subagents') + subagents_dir = os.path.join(output_dir, '.ms_agent', 'subagents') os.makedirs(subagents_dir, exist_ok=True) safe_id = self._call_id.replace('/', '_').replace('\\', '_') self._path: str = os.path.join(subagents_dir, diff --git a/ms_agent/utils/utils.py b/ms_agent/utils/utils.py index 91d9c6d42..2410aea3f 100644 --- a/ms_agent/utils/utils.py +++ b/ms_agent/utils/utils.py @@ -241,10 +241,21 @@ def read_history(output_dir: str, task: str): """ from ms_agent.config import Config from ms_agent.llm import Message + from .constants import LEGACY_MEMORY_DIR cache_dir = os.path.join(output_dir, DEFAULT_MEMORY_DIR) - os.makedirs(cache_dir, exist_ok=True) config_file = os.path.join(cache_dir, f'{task}.yaml') message_file = os.path.join(cache_dir, f'{task}.json') + # Back-compat: fall back to the legacy /.memory cache when the + # new .ms_agent/memory location has no history yet. + if not os.path.exists(message_file) and not os.path.exists(config_file): + legacy_dir = os.path.join(output_dir, LEGACY_MEMORY_DIR) + legacy_msg = os.path.join(legacy_dir, f'{task}.json') + if os.path.exists(legacy_msg) or os.path.exists( + os.path.join(legacy_dir, f'{task}.yaml')): + cache_dir = legacy_dir + config_file = os.path.join(cache_dir, f'{task}.yaml') + message_file = os.path.join(cache_dir, f'{task}.json') + os.makedirs(cache_dir, exist_ok=True) config = None messages = None if os.path.exists(config_file): @@ -651,8 +662,11 @@ def install_package(package_name: str, package_name = f'{package_name}[{extend_module}]' if not is_package_installed(import_name): + # Suppress pip's stdout ("Requirement already satisfied ...") so it does + # not pollute interactive UIs; stderr is kept for real errors. subprocess.check_call( - [sys.executable, '-m', 'pip', 'install', package_name]) + [sys.executable, '-m', 'pip', 'install', package_name], + stdout=subprocess.DEVNULL) logger.info(f'Package {package_name} installed successfully.') else: logger.info(f'Package {import_name} is already installed.') diff --git a/requirements/framework.txt b/requirements/framework.txt index 96a410faf..354db5e74 100644 --- a/requirements/framework.txt +++ b/requirements/framework.txt @@ -14,8 +14,10 @@ omegaconf openai pandas pillow +prompt_toolkit pyyaml pytz requests +rich sentence-transformers typing_extensions diff --git a/tests/command/test_interactive_input.py b/tests/command/test_interactive_input.py index 913932325..334670362 100644 --- a/tests/command/test_interactive_input.py +++ b/tests/command/test_interactive_input.py @@ -173,6 +173,8 @@ def _make_callback_agent(config): # main's _get_command_router() -> _register_plugin_commands() reads this; # None makes it a no-op (real agents set it in __init__). agent._plugin_runtime = None + agent._event_sink = None + agent._input_source = None return agent diff --git a/tests/command/test_new_cmds.py b/tests/command/test_new_cmds.py index 15458130f..d7a9bee8b 100644 --- a/tests/command/test_new_cmds.py +++ b/tests/command/test_new_cmds.py @@ -172,7 +172,8 @@ async def test_switch_model_persists_to_project_patch(self, tmp_path): assert cfg_file.read_text(encoding='utf-8') == yaml_text # The override landed in the project patch, which from_task merges back. - patch_file = tmp_path / '.ms-agent' / 'config.yaml' + # Writer now targets the new .ms_agent/ dir (resolver reads both). + patch_file = tmp_path / '.ms_agent' / 'config.yaml' assert patch_file.exists() patch_cfg = OmegaConf.load(str(patch_file)) assert patch_cfg.llm.model == 'qwen3.7-max' diff --git a/tests/config/test_config.py b/tests/config/test_config.py index 70bd35216..2ef4f4a94 100644 --- a/tests/config/test_config.py +++ b/tests/config/test_config.py @@ -31,23 +31,42 @@ def test_safe_get_config(self): config, 'tools.file_system.system_for_abbreviations') is None) @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') - def test_from_task_merges_project_patch(self): - # A project patch (written by /model) wins over the committed YAML, - # while the source file stays untouched. + def test_from_task_does_not_leak_config_dir_patch(self): + # A patch in the config file's own directory must NOT be merged by + # from_task: overrides are anchored to the work dir, not the config + # file's folder, so running a shared/template config never picks up + # (or scatters) overrides in that folder. with tempfile.TemporaryDirectory() as d: with open(os.path.join(d, 'agent.yaml'), 'w') as f: f.write('llm:\n service: openai\n model: base-model\n') - patch_dir = os.path.join(d, '.ms-agent') + patch_dir = os.path.join(d, '.ms_agent') os.makedirs(patch_dir) with open(os.path.join(patch_dir, 'config.yaml'), 'w') as f: f.write('llm:\n model: override-model\n') - # Isolate from pytest's argv, which Config.parse_args() scans. with patch.object(sys, 'argv', ['ms-agent']): config = Config.from_task(d) - self.assertEqual('override-model', config.llm.model) - # Untouched base key is preserved through the merge. - self.assertEqual('openai', config.llm.service) + # from_task leaves the committed model intact (no config-dir merge). + self.assertEqual('base-model', config.llm.model) + + def test_work_dir_patch_merges(self): + # The project patch is anchored to the work dir and applied by the + # agent (BaseAgent.__init__ via ConfigResolver._load_project_patch). + from omegaconf import OmegaConf + + from ms_agent.config.resolver import ConfigResolver + with tempfile.TemporaryDirectory() as work: + patch_dir = os.path.join(work, '.ms_agent') + os.makedirs(patch_dir) + with open(os.path.join(patch_dir, 'config.yaml'), 'w') as f: + f.write('llm:\n model: override-model\n') + base = OmegaConf.create( + {'llm': {'service': 'openai', 'model': 'base-model'}}) + patch_cfg = ConfigResolver()._load_project_patch(work) + self.assertIsNotNone(patch_cfg) + merged = OmegaConf.merge(base, patch_cfg) + self.assertEqual('override-model', merged.llm.model) + self.assertEqual('openai', merged.llm.service) @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') def test_from_task_without_patch(self): @@ -58,6 +77,14 @@ def test_from_task_without_patch(self): config = Config.from_task(d) self.assertEqual('base-model', config.llm.model) + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_parse_args_ignores_bare_separator(self): + # A bare '--' separator must not be parsed as an empty-key override + # (without the len(key) > 2 guard this yields {'': 'v'}). + with patch.object(sys, 'argv', ['ms-agent', '--', '--', 'v']): + parsed = Config.parse_args() + self.assertNotIn('', parsed) + if __name__ == '__main__': unittest.main() diff --git a/tests/config/test_env_override.py b/tests/config/test_env_override.py new file mode 100644 index 000000000..7d037fa7e --- /dev/null +++ b/tests/config/test_env_override.py @@ -0,0 +1,44 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Config.from_task matches config leaf keys against env names case-insensitively +to allow overrides (e.g. OPENAI_API_KEY -> llm.openai_api_key). Ambient shell +vars (PATH/HOME/...) must be excluded from that source, else a config key like +``path`` (skills sources[].path) is silently clobbered by the shell's $PATH.""" +import sys +from unittest.mock import patch + +from ms_agent.config import Config + + +def _cfg_dir(tmp_path, body): + (tmp_path / 'agent.yaml').write_text(body) + return str(tmp_path) + + +def test_path_key_not_clobbered_by_env_PATH(tmp_path, monkeypatch): + monkeypatch.setenv('PATH', '/usr/bin:/bin:/sentinel/from/shell') + d = _cfg_dir(tmp_path, ( + 'llm:\n service: openai\n model: m\n' + 'skills:\n sources:\n - type: local\n' + ' path: /marker/skills/dir\n')) + with patch.object(sys, 'argv', ['ms-agent']): + cfg = Config.from_task(d) + assert cfg.skills.sources[0].path == '/marker/skills/dir' + + +def test_home_key_not_clobbered_by_env_HOME(tmp_path, monkeypatch): + monkeypatch.setenv('HOME', '/Users/somebody') + d = _cfg_dir(tmp_path, + 'llm:\n service: openai\n model: m\n home: /keep/this\n') + with patch.object(sys, 'argv', ['ms-agent']): + cfg = Config.from_task(d) + assert cfg.llm.home == '/keep/this' + + +def test_non_shell_env_override_still_applies(tmp_path, monkeypatch): + # A non-blocklisted env var must still override a same-named config key — + # the feature the blocklist must NOT break. + monkeypatch.setenv('MODEL', 'env-model-xyz') + d = _cfg_dir(tmp_path, 'llm:\n service: openai\n model: file-model\n') + with patch.object(sys, 'argv', ['ms-agent']): + cfg = Config.from_task(d) + assert cfg.llm.model == 'env-model-xyz' diff --git a/tests/config/test_model_settings.py b/tests/config/test_model_settings.py new file mode 100644 index 000000000..953fd6b0d --- /dev/null +++ b/tests/config/test_model_settings.py @@ -0,0 +1,51 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +import json + +from ms_agent.config.model_settings import ModelSettingsManager + + +def test_add_list_remove_provider(tmp_path): + m = ModelSettingsManager(global_dir=str(tmp_path)) + m.add_provider('acme', name='Acme', protocol='openai', + api_key='k', base_url='https://acme/v1', models=['a-1']) + customs = m.list_custom_providers() + assert customs['acme']['base_url'] == 'https://acme/v1' + assert 'a-1' in customs['acme']['models'] + # builtin + custom listed together + ids = {p['id'] for p in m.list_providers()} + assert 'acme' in ids and 'openai' in ids and 'deepseek' in ids + m.remove_provider('acme') + assert 'acme' not in m.list_custom_providers() + + +def test_models_and_default(tmp_path): + m = ModelSettingsManager(global_dir=str(tmp_path)) + m.add_provider('acme', protocol='openai') + m.add_model('acme', 'a-2') + assert 'a-2' in m.list_custom_providers()['acme']['models'] + m.set_default_model('a-2', provider='acme') + assert m.get_default_model() == 'acme/a-2' + m.remove_model('acme', 'a-2') + assert 'a-2' not in m.list_custom_providers()['acme']['models'] + + +def test_preserves_other_sections(tmp_path): + p = tmp_path / 'settings.json' + p.write_text(json.dumps({'theme': 'dark', 'llm': {'provider': 'x'}})) + m = ModelSettingsManager(global_dir=str(tmp_path)) + m.add_provider('acme', protocol='openai') + data = json.loads(p.read_text()) + assert data['theme'] == 'dark' and data['llm']['provider'] == 'x' + assert 'acme' in data['providers'] + + +def test_resolver_consumes_default_model(): + from ms_agent.config.resolver import ConfigResolver + cfg = ConfigResolver._settings_to_agent_config( + {'default_model': 'deepseek/deepseek-chat'}) + assert cfg.llm.service == 'deepseek' + assert cfg.llm.model == 'deepseek-chat' + # explicit llm.model wins over default_model + cfg2 = ConfigResolver._settings_to_agent_config( + {'llm': {'model': 'pinned'}, 'default_model': 'deepseek/x'}) + assert cfg2.llm.model == 'pinned' diff --git a/tests/llm/test_continuation_guard.py b/tests/llm/test_continuation_guard.py new file mode 100644 index 000000000..c65005691 --- /dev/null +++ b/tests/llm/test_continuation_guard.py @@ -0,0 +1,48 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""PR#920 review #2/#3: continuation (partial/prefix) must only be attempted +when the provider supports it (``continue_gen_mode`` set). Otherwise a truncated +response (``finish_reason='length'``) on a strict OpenAI-compatible API would +send ``partial``/``prefix`` fields + consecutive assistant messages -> 400.""" +from unittest.mock import MagicMock, patch + +from ms_agent.llm.transport.openai_compat import OpenAICompatTransport +from ms_agent.llm.utils import Message + + +def _transport(continue_gen_mode): + # Bypass __init__ so no openai client / network is created. + t = OpenAICompatTransport.__new__(OpenAICompatTransport) + t.continue_gen_mode = continue_gen_mode + t._continue_flag = 'prefix' if continue_gen_mode == 'prefix' else 'partial' + return t + + +def _length_completion(): + c = MagicMock() + c.choices[0].finish_reason = 'length' # truncated + return c + + +def test_no_continuation_when_mode_none(): + """continue_gen_mode=None (standard OpenAI/Gemini) -> never continue.""" + t = _transport(None) + msgs = [Message(role='user', content='hi')] + with patch.object(t, '_format_output_message', + return_value=Message(role='assistant', content='x')), \ + patch.object(t, '_call_llm_for_continue_gen') as cont: + t._continue_generate(msgs, _length_completion(), tools=None) + cont.assert_not_called() + + +def test_continuation_when_mode_set(): + """A provider that supports continuation still continues on 'length'.""" + t = _transport('partial') + msgs = [Message(role='user', content='hi')] + stop = MagicMock() + stop.choices[0].finish_reason = 'stop' # ends the recursion + with patch.object(t, '_format_output_message', + return_value=Message(role='assistant', content='x')), \ + patch.object( + t, '_call_llm_for_continue_gen', return_value=stop) as cont: + t._continue_generate(msgs, _length_completion(), tools=None) + cont.assert_called_once() diff --git a/tests/llm/test_provider_router_live.py b/tests/llm/test_provider_router_live.py index c82f02761..ba92e51ab 100644 --- a/tests/llm/test_provider_router_live.py +++ b/tests/llm/test_provider_router_live.py @@ -49,8 +49,8 @@ PROVIDERS = { 'modelscope': ('Qwen/Qwen3-235B-A22B-Instruct-2507', 'MODELSCOPE_API_KEY'), 'dashscope': ('qwen3.7-plus', 'DASHSCOPE_API_KEY'), - 'deepseek': ('deepseek-v4-flash', 'DEEPSEEK_API_KEY'), - 'zhipu': ('glm-4.6', 'GLM_API_KEY'), + 'deepseek': ('deepseek-chat', 'DEEPSEEK_API_KEY'), + 'zhipu': ('glm-4.5', 'GLM_API_KEY'), 'kimi': ('moonshot-v1-8k', 'KIMI_API_KEY'), 'minimax': ('MiniMax-M2', 'MINIMAX_API_KEY'), 'openrouter': ('qwen/qwen3.7-plus', 'OpenRouter_API_KEY'), @@ -100,11 +100,24 @@ def _run(self, service): pass self.assertTrue(chunk and chunk.content, f'{service}: empty stream') - # tool call + # tool call (non-stream) llm = LLM.from_config(_config(service, model)) res = llm.generate(messages=_msgs('请创建一个名为 demo 的目录。'), tools=TOOLS) self.assertTrue(res.tool_calls, f'{service}: no tool call') + # tool call (stream) — the streamed tool_call merge must produce a + # complete name + arguments, not a partial fragment. + llm = LLM.from_config(_config(service, model, stream=True)) + chunk = None + for chunk in llm.generate( + messages=_msgs('请创建一个名为 demo 的目录。'), tools=TOOLS): + pass + self.assertTrue(chunk and chunk.tool_calls, + f'{service}: no streamed tool call') + tc = chunk.tool_calls[0] + self.assertTrue(tc.get('tool_name') and tc.get('arguments'), + f'{service}: incomplete streamed tool call') + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') def test_modelscope(self): self._run('modelscope') @@ -133,6 +146,19 @@ def test_minimax(self): def test_openrouter(self): self._run('openrouter') + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') + def test_reasoning_content(self): + """A reasoning model surfaces reasoning_content separate from content.""" + if not os.getenv('DEEPSEEK_API_KEY'): + self.skipTest('needs DEEPSEEK_API_KEY') + model = os.getenv('DEEPSEEK_REASONER_TEST_MODEL', 'deepseek-reasoner') + llm = LLM.from_config(_config('deepseek', model)) + res = llm.generate(messages=_msgs('2+2 等于几?简要思考。'), tools=None) + self.assertTrue(res.content, 'reasoner: empty content') + self.assertTrue(getattr(res, 'reasoning_content', ''), + 'reasoner: no reasoning_content') + self.assertNotIn('', res.content) + @unittest.skipUnless(test_level() >= 0, 'skip test in current test level') def test_continue_gen_accumulates(self): if not os.getenv('DASHSCOPE_API_KEY'): diff --git a/tests/permission/test_parallel_permission.py b/tests/permission/test_parallel_permission.py new file mode 100644 index 000000000..421442a26 --- /dev/null +++ b/tests/permission/test_parallel_permission.py @@ -0,0 +1,45 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Parallel tool calls (ToolManager.parallel_call_tool → asyncio.gather) must +not invoke the interactive permission handler concurrently: N prompts fighting +over one terminal deadlocks. The enforcer serializes asks.""" +import asyncio + +import pytest + +from ms_agent.permission.config import PermissionConfig +from ms_agent.permission.enforcer import PermissionEnforcer +from ms_agent.permission.handler import PermissionAction, PermissionResponse +from ms_agent.permission.memory import PermissionMemory + + +class _ConcurrencyProbe: + def __init__(self): + self.in_flight = 0 + self.max_in_flight = 0 + self.calls = 0 + + async def ask(self, tool_name, tool_args, context, suggestions=None): + self.in_flight += 1 + self.calls += 1 + self.max_in_flight = max(self.max_in_flight, self.in_flight) + await asyncio.sleep(0.02) # simulate a human deciding + self.in_flight -= 1 + return PermissionResponse(action=PermissionAction.ALLOW_ONCE) + + +@pytest.mark.asyncio +async def test_parallel_asks_are_serialized(tmp_path): + cfg = PermissionConfig.from_dict({'mode': 'restricted'}) # empty wl -> ask + handler = _ConcurrencyProbe() + enf = PermissionEnforcer( + config=cfg, handler=handler, + memory=PermissionMemory(project_path=str(tmp_path))) + + # 5 concurrent checks, like 5 parallel tool calls in one round. + results = await asyncio.gather( + *[enf.check('some_tool', {'i': i}) for i in range(5)]) + + assert all(r.action == 'allow' for r in results) + assert handler.calls == 5 + # The decisive assertion: asks never overlapped (was 5 before the fix). + assert handler.max_in_flight == 1 diff --git a/tests/permission/test_safety_ask_force.py b/tests/permission/test_safety_ask_force.py new file mode 100644 index 000000000..188c58144 --- /dev/null +++ b/tests/permission/test_safety_ask_force.py @@ -0,0 +1,52 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""REVIEW P1-2: a SafetyGuard `ask` (passed as force_decision) must reach the +handler even when a whitelist/memory entry would otherwise allow the tool.""" +import pytest + +from ms_agent.permission.config import PermissionConfig +from ms_agent.permission.enforcer import PermissionDecision, PermissionEnforcer +from ms_agent.permission.memory import PermissionMemory + + +class _RecordingHandler: + def __init__(self): + self.asked = False + + async def ask(self, tool_name, tool_args, context, suggestions=None): + self.asked = True + from ms_agent.permission.handler import (PermissionAction, + PermissionResponse) + return PermissionResponse(action=PermissionAction.DENY) + + +@pytest.mark.asyncio +async def test_force_ask_not_bypassed_by_whitelist(tmp_path): + cfg = PermissionConfig.from_dict({ + 'mode': 'interactive', + 'whitelist': ['code_executor---shell_executor:*'], + }) + handler = _RecordingHandler() + enf = PermissionEnforcer( + config=cfg, handler=handler, + memory=PermissionMemory(project_path=str(tmp_path))) + force = PermissionDecision(action='ask', reason='safety') + out = await enf.check( + 'code_executor---shell_executor', {'command': 'ls'}, + force_decision=force) + # whitelist would allow, but the safety force_decision routed to the handler + assert handler.asked is True + assert out.action == 'deny' + + +@pytest.mark.asyncio +async def test_whitelist_allows_without_force(tmp_path): + cfg = PermissionConfig.from_dict({ + 'mode': 'interactive', + 'whitelist': ['code_executor---shell_executor:*'], + }) + handler = _RecordingHandler() + enf = PermissionEnforcer( + config=cfg, handler=handler, + memory=PermissionMemory(project_path=str(tmp_path))) + out = await enf.check('code_executor---shell_executor', {'command': 'ls'}) + assert handler.asked is False and out.action == 'allow' diff --git a/tests/permission/test_set_mode.py b/tests/permission/test_set_mode.py new file mode 100644 index 000000000..7126ce72e --- /dev/null +++ b/tests/permission/test_set_mode.py @@ -0,0 +1,52 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""LLMAgent.set_permission_mode: live mode switch used by the TUI /permission.""" +import pytest + +from ms_agent.agent.llm_agent import LLMAgent +from ms_agent.permission.config import PermissionConfig + + +def _agent_with_enforcer(): + agent = LLMAgent.__new__(LLMAgent) + cfg = PermissionConfig() # default mode='auto' + + class _Enf: + pass + + class _TM: + pass + + enf = _Enf() + enf._config = cfg + tm = _TM() + tm._permission_mode = 'auto' + tm._permission_config = cfg + tm._permission_enforcer = enf + agent.tool_manager = tm + return agent, tm, enf + + +def test_switch_updates_toolmanager_and_enforcer(): + agent, tm, enf = _agent_with_enforcer() + assert agent.set_permission_mode('strict') == 'strict' + assert tm._permission_mode == 'strict' + assert tm._permission_config.mode == 'strict' + assert enf._config.mode == 'strict' + + +def test_restricted_normalizes_to_interactive(): + agent, tm, enf = _agent_with_enforcer() + assert agent.set_permission_mode('restricted') == 'interactive' + assert enf._config.mode == 'interactive' + + +def test_invalid_mode_raises(): + agent, _, _ = _agent_with_enforcer() + with pytest.raises(ValueError): + agent.set_permission_mode('bogus') + + +def test_no_toolmanager_is_safe(): + agent = LLMAgent.__new__(LLMAgent) + agent.tool_manager = None + assert agent.set_permission_mode('auto') == 'auto' # no crash diff --git a/tests/project/conftest.py b/tests/project/conftest.py new file mode 100644 index 000000000..2997ab7ba --- /dev/null +++ b/tests/project/conftest.py @@ -0,0 +1,14 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Isolate the global ms-agent home for project tests. + +SessionManager (and the runtime SessionLog) place sessions under the global +``~/.ms_agent/projects/<...>`` root (paths.py). Redirect ``MS_AGENT_HOME`` to a +per-test temp dir so tests never read or pollute the real user home. +""" +import pytest + + +@pytest.fixture(autouse=True) +def _isolate_ms_agent_home(tmp_path, monkeypatch): + monkeypatch.setenv('MS_AGENT_HOME', str(tmp_path / '_ms_home')) + yield diff --git a/tests/project/test_manager.py b/tests/project/test_manager.py index 6a59f490e..7812359ac 100644 --- a/tests/project/test_manager.py +++ b/tests/project/test_manager.py @@ -81,10 +81,61 @@ def test_workspace_dir_created(self, pm): def test_sessions_dir_created(self, pm): project = pm.create(name='WithSessions') from pathlib import Path - meta_dir = pm._projects_root / project.id / '.ms-agent' - assert (meta_dir / 'sessions').is_dir() + assert (pm._projects_root / project.id / 'sessions').is_dir() def test_project_is_frozen(self, pm): project = pm.create(name='Frozen') with pytest.raises(AttributeError): project.name = 'Mutated' + + def test_create_init_workspace_false_skips_workspace(self, pm, tmp_path): + from pathlib import Path + custom = tmp_path / 'no_ws' + project = pm.create( + name='NoWs', path=str(custom), init_workspace=False) + assert not (Path(project.path) / 'workspace').exists() + + # -- open_folder (Codex "use an existing folder") -- + + def test_open_folder_id_is_path_key(self, pm, tmp_path): + from ms_agent.project.paths import project_key + folder = tmp_path / 'my-repo' + folder.mkdir() + project = pm.open_folder(str(folder)) + assert project.id == project_key(str(folder)) + assert project.path == str(folder.resolve()) + assert project.name == 'my-repo' # defaults to the folder basename + + def test_open_folder_does_not_create_workspace(self, pm, tmp_path): + from pathlib import Path + folder = tmp_path / 'existing-repo' + folder.mkdir() + pm.open_folder(str(folder)) + # The existing folder must stay clean — no injected workspace/. + assert not (Path(folder) / 'workspace').exists() + + def test_open_folder_dedups_same_path(self, pm, tmp_path): + folder = tmp_path / 'repo' + folder.mkdir() + p1 = pm.open_folder(str(folder), name='First') + p2 = pm.open_folder(str(folder), name='Second') + assert p1.id == p2.id + # Reopening returns the existing project, not a duplicate. + assert p2.name == 'First' + matching = [p for p in pm.list() if p.path == str(folder.resolve())] + assert len(matching) == 1 + + def test_open_folder_appears_in_list(self, pm, tmp_path): + folder = tmp_path / 'listed-repo' + folder.mkdir() + project = pm.open_folder(str(folder)) + assert project.id in [p.id for p in pm.list()] + + def test_open_folder_roundtrips_via_get(self, pm, tmp_path): + folder = tmp_path / 'gettable' + folder.mkdir() + project = pm.open_folder(str(folder), instruction='be terse') + again = pm.get(project.id) + assert again is not None + assert again.path == project.path + assert again.instruction == 'be terse' diff --git a/tests/project/test_paths.py b/tests/project/test_paths.py new file mode 100644 index 000000000..af0f06423 --- /dev/null +++ b/tests/project/test_paths.py @@ -0,0 +1,47 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Every framework-internal path resolves under /.ms_agent/ (or the +global ~/.ms_agent), so a work dir never accumulates scattered files.""" +from pathlib import Path + +from ms_agent.project import paths + + +def test_work_local_helpers_under_ms_agent(tmp_path): + w = str(tmp_path) + internal = str(paths.local_internal_dir(w)) + for fn in (paths.memory_dir, paths.snapshots_dir, paths.artifacts_dir, + paths.index_dir, paths.locks_dir, paths.tmp_dir, + paths.subagents_dir, paths.web_search_dir): + assert str(fn(w)).startswith(internal), fn.__name__ + assert str(paths.search_index_dir(w, 'rag')).startswith(internal) + assert str(paths.stats_file(w)).startswith(internal) + + +def test_global_helpers_under_ms_agent_home(tmp_path, monkeypatch): + monkeypatch.setenv('MS_AGENT_HOME', str(tmp_path / 'home')) + home = str((tmp_path / 'home').resolve()) + assert str(paths.global_projects_root()).startswith(home) + assert str(paths.global_logs_dir()).startswith(home) + assert str(paths.global_project_dir('/some/work')).startswith(home) + + +def test_project_internal_file_prefers_new_then_legacy(tmp_path): + proj = tmp_path / 'proj' + # nothing exists -> returns the new .ms_agent path + p = paths.project_internal_file(proj, 'mcp.json') + assert p.parent.name == '.ms_agent' + # only legacy exists -> returns legacy + legacy = proj / '.ms-agent' + legacy.mkdir(parents=True) + (legacy / 'mcp.json').write_text('{}') + assert paths.project_internal_file(proj, 'mcp.json').parent.name == '.ms-agent' + # new exists -> prefers new + new = proj / '.ms_agent' + new.mkdir(parents=True) + (new / 'mcp.json').write_text('{}') + assert paths.project_internal_file(proj, 'mcp.json').parent.name == '.ms_agent' + + +def test_project_key_stable_and_readable(): + assert paths.project_key('/Users/x/proj') == paths.project_key('/Users/x/proj') + assert 'proj' in paths.project_key('/Users/x/proj') diff --git a/tests/project/test_workspace_upload.py b/tests/project/test_workspace_upload.py new file mode 100644 index 000000000..0d5d4e29d --- /dev/null +++ b/tests/project/test_workspace_upload.py @@ -0,0 +1,58 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +import io +import zipfile + +import pytest + +from ms_agent.project.workspace import Workspace + + +def test_write_read_bytes(tmp_path): + ws = Workspace(str(tmp_path / 'ws')) + ws.write_bytes('sub/data.bin', b'\x00\x01\x02') + assert ws.read_bytes('sub/data.bin') == b'\x00\x01\x02' + + +def test_save_upload_stream_and_bytes(tmp_path): + ws = Workspace(str(tmp_path / 'ws')) + rel = ws.save_upload('report.pdf', io.BytesIO(b'PDFDATA'), subdir='uploads') + assert ws.read_bytes(rel) == b'PDFDATA' + # basename-only: a filename with dir components is stripped + rel2 = ws.save_upload('../../evil.txt', b'x') + assert 'evil.txt' in rel2 and '..' not in rel2 + + +def test_save_upload_traversal_blocked(tmp_path): + ws = Workspace(str(tmp_path / 'ws')) + with pytest.raises(PermissionError): + ws.save_upload('ok.txt', b'x', subdir='../../etc') + + +def test_zip_download(tmp_path): + ws = Workspace(str(tmp_path / 'ws')) + ws.write_file('a.txt', 'A') + ws.write_file('d/b.txt', 'B') + data = ws.zip_download('.') + with zipfile.ZipFile(io.BytesIO(data)) as zf: + names = set(zf.namelist()) + assert any(n.endswith('a.txt') for n in names) + assert any(n.endswith('b.txt') for n in names) + + +def test_zip_download_skips_external_symlink(tmp_path): + # A symlink planted in the workspace pointing outside must NOT be followed + # and packaged (arbitrary-file-read / info disclosure via zip_download). + secret = tmp_path / 'secret.txt' + secret.write_text('TOP-SECRET') + ws_dir = tmp_path / 'ws' + ws = Workspace(str(ws_dir)) + ws.write_file('safe.txt', 'ok') + (ws_dir / 'leak.txt').symlink_to(secret) # escapes the workspace + + data = ws.zip_download('.') + with zipfile.ZipFile(io.BytesIO(data)) as zf: + names = zf.namelist() + blob = b''.join(zf.read(n) for n in names) + assert any(n.endswith('safe.txt') for n in names) + assert not any('leak' in n for n in names) # symlink entry skipped + assert b'TOP-SECRET' not in blob # external content not leaked diff --git a/tests/tui/test_managed_config.py b/tests/tui/test_managed_config.py new file mode 100644 index 000000000..f4289a01e --- /dev/null +++ b/tests/tui/test_managed_config.py @@ -0,0 +1,65 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Bridge managed .ms_agent/{mcp,skills}.json config into the agent runtime.""" +import json + +from omegaconf import OmegaConf + +from ms_agent.config import MCPConfigManager +from ms_agent.config.skills_manager import SkillsConfigManager +from ms_agent.tui.managed_config import (merge_skills_into_config, + resolve_mcp_config) + + +# ── MCP ──────────────────────────────────────────────────────────────────── + + +def test_mcp_enabled_only_and_meta_stripped(tmp_path): + home = str(tmp_path / 'home') + mm = MCPConfigManager(global_root=home, project_root=None) + mm.add('a', {'type': 'streamable_http', 'url': 'http://a'}, scope='global') + mm.add('b', {'type': 'streamable_http', 'url': 'http://b'}, scope='global') + mm.set_enabled('b', False, scope='global') + r = resolve_mcp_config(home, None, None) + assert set(r['mcpServers']) == {'a'} # disabled 'b' dropped + # meta fields (enabled/meta/source) stripped — only the server config left + assert r['mcpServers']['a'] == {'type': 'streamable_http', 'url': 'http://a'} + + +def test_mcp_none_when_empty(tmp_path): + assert resolve_mcp_config(str(tmp_path / 'home'), None, None) is None + + +def test_mcp_explicit_file_wins(tmp_path): + home = str(tmp_path / 'home') + MCPConfigManager(global_root=home, project_root=None).add( + 'a', {'type': 'streamable_http', 'url': 'http://from-json'}, + scope='global') + ef = tmp_path / 'explicit.json' + ef.write_text(json.dumps( + {'mcpServers': {'a': {'type': 'streamable_http', 'url': 'http://EXPLICIT'}}})) + r = resolve_mcp_config(home, None, str(ef)) + assert r['mcpServers']['a']['url'] == 'http://EXPLICIT' # explicit wins last + + +# ── Skill ────────────────────────────────────────────────────────────────── + + +def test_skills_sources_appended_and_disabled_unioned(tmp_path): + home = str(tmp_path / 'home') + sk = SkillsConfigManager(global_dir=home) + sk.add_source('/abs/managed-skills', scope='global') + sk.set_skill_enabled('foo', False, scope='global') + cfg = OmegaConf.create( + {'skills': {'sources': [{'type': 'local', 'path': '/existing'}]}}) + cfg = merge_skills_into_config(cfg, home, None) + srcs = OmegaConf.to_container(cfg.skills.sources) + assert {'type': 'local', 'path': '/existing'} in srcs # existing preserved + assert any(str(s.get('path', '')).endswith('managed-skills') + for s in srcs) # managed source appended (string→structured) + assert 'foo' in list(cfg.skills.disabled) # disabled unioned + + +def test_skills_noop_when_empty(tmp_path): + cfg = OmegaConf.create({'llm': {'model': 'x'}}) + out = merge_skills_into_config(cfg, str(tmp_path / 'home'), None) + assert not getattr(out, 'skills', None) # nothing added diff --git a/tests/tui/test_renderer.py b/tests/tui/test_renderer.py new file mode 100644 index 000000000..2413ee38d --- /dev/null +++ b/tests/tui/test_renderer.py @@ -0,0 +1,121 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""RichEventSink: renders the AgentEvent stream and tracks status state.""" +from io import StringIO + +from rich.console import Console + +from ms_agent.tui.renderer import RichEventSink +from ms_agent.tui.state import TuiState +from ms_agent.ui.events import (ContentDelta, ContentEnd, ErrorRaised, Notice, + PlanEntry, PlanUpdated, ReasoningDelta, + ReasoningEnded, ReasoningStarted, + ToolCallCompleted, ToolCallStarted, TurnCompleted, + TurnStarted, UsageInfo) + + +def _sink(): + console = Console(file=StringIO(), force_terminal=False, width=80) + state = TuiState() + return RichEventSink(console, state), console, state + + +def _out(console) -> str: + return console.file.getvalue() + + +def test_content_stream_renders_text(): + sink, console, _ = _sink() + sink.emit(ContentDelta('Hello ')) + sink.emit(ContentDelta('world')) + sink.emit(ContentEnd()) + assert 'Hello world' in _out(console) + + +def test_tool_call_and_result_render(): + # Compact Codex-style lines: "• Search modelscope" then " └ found it". + sink, console, _ = _sink() + sink.emit(ToolCallStarted(call_id='c1', name='web_search---exa_search', + arguments={'query': 'modelscope'})) + sink.emit(ToolCallCompleted(call_id='c1', name='web_search---exa_search', + result='found it')) + out = _out(console) + assert '•' in out and 'Search' in out and 'modelscope' in out + assert '└' in out and 'found it' in out + + +def test_tool_error_render(): + sink, console, _ = _sink() + sink.emit(ToolCallStarted(call_id='c2', name='code_executor---shell', + arguments={'command': 'bad'})) + sink.emit(ToolCallCompleted(call_id='c2', name='code_executor---shell', + result='', error='command not found')) + assert 'error' in _out(console) and 'command not found' in _out(console) + + +def test_config_output_syntax_highlighted(): + # /config-style YAML notice renders (no crash; content present). + sink, console, _ = _sink() + sink.emit(Notice(level='info', + text='llm:\n model: qwen\n service: openai\ntag: x')) + out = _out(console) + assert 'model' in out and 'qwen' in out + + +def test_turn_completed_updates_status_state(): + sink, _, state = _sink() + sink.emit(TurnCompleted(usage=UsageInfo( + total_prompt_tokens=100, total_completion_tokens=50))) + assert state.total_prompt_tokens == 100 + assert state.total_completion_tokens == 50 + assert state.total_tokens == 150 + + +def test_reasoning_renders_thinking_block(): + sink, console, _ = _sink() + sink.emit(ReasoningStarted()) + sink.emit(ReasoningDelta('let me consider')) + sink.emit(ReasoningEnded()) + out = _out(console) + assert 'Thinking' in out and 'let me consider' in out + + +def test_gap_between_tool_and_thinking(): + # A blank line must separate a tool result from a following Thinking block. + sink, console, _ = _sink() + sink.emit(ToolCallStarted(call_id='c', name='skills---skill_view', + arguments={})) + sink.emit(ToolCallCompleted(call_id='c', name='skills---skill_view', + result='done')) + sink.emit(ReasoningStarted()) + sink.emit(ReasoningDelta('hmm')) + sink.emit(ReasoningEnded()) + lines = _out(console).split('\n') + ti = next(i for i, ln in enumerate(lines) if '└' in ln) + thi = next(i for i, ln in enumerate(lines) if 'Thinking' in ln) + assert any(lines[j].strip() == '' for j in range(ti + 1, thi)) + + +def test_error_renders_message(): + sink, console, _ = _sink() + sink.emit(ErrorRaised(message='boom')) + assert 'boom' in _out(console) + + +def test_notice_and_plan_render(): + sink, console, _ = _sink() + sink.emit(Notice(level='success', text='saved')) + sink.emit(PlanUpdated(entries=[PlanEntry('do X', 'completed')])) + out = _out(console) + assert 'saved' in out and 'do X' in out + + +def test_unhandled_event_is_ignored(): + sink, console, _ = _sink() + # No _on_turn_started handler exists — emit must be a safe no-op. + sink.emit(TurnStarted(turn_id='t1')) + assert _out(console) == '' + + +def test_finalize_is_safe_without_live(): + sink, _, _ = _sink() + sink.finalize() # no active Live — must not raise diff --git a/tests/tui/test_select.py b/tests/tui/test_select.py new file mode 100644 index 000000000..ff7b23e41 --- /dev/null +++ b/tests/tui/test_select.py @@ -0,0 +1,73 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Inline selection menu key handling (driven via a prompt_toolkit pipe). + +Validates the arrow-key / number-key / enter / cancel logic without a real +terminal, so the permission menu's behavior is regression-guarded. +""" +import asyncio + +from prompt_toolkit.application import create_app_session +from prompt_toolkit.input import create_pipe_input +from prompt_toolkit.output import DummyOutput + +from ms_agent.tui.select import _menu_async + +OPTS = ['Allow once', 'Allow for this session', 'Always allow', 'Deny'] + + +def _run(keys: str, default: int = 0): + async def go(): + with create_pipe_input() as inp: + with create_app_session(input=inp, output=DummyOutput()): + inp.send_text(keys) + return await _menu_async(OPTS, default) + + return asyncio.run(go()) + + +def test_enter_picks_default(): + assert _run('\r') == 0 + + +def test_enter_picks_given_default(): + assert _run('\r', default=2) == 2 + + +def test_down_then_enter(): + assert _run('\x1b[B\r') == 1 # ↓, Enter + + +def test_down_wraps_from_last_to_first(): + # 4 options: ↓×4 wraps back to index 0 + assert _run('\x1b[B\x1b[B\x1b[B\x1b[B\r') == 0 + + +def test_up_then_enter_wraps(): + assert _run('\x1b[A\r') == len(OPTS) - 1 # ↑ from 0 wraps to last + + +def test_tab_moves_down(): + assert _run('\t\r') == 1 + + +def test_number_key_jumps_and_selects(): + assert _run('3') == 2 + + +def test_ctrl_c_cancels_to_none(): + assert _run('\x03') is None + + +def _run_with_header(keys: str, header: str): + async def go(): + with create_pipe_input() as inp: + with create_app_session(input=inp, output=DummyOutput()): + inp.send_text(keys) + return await _menu_async(OPTS, 0, header) + + return asyncio.run(go()) + + +def test_header_does_not_break_selection(): + # A multi-line header (tool + args) renders above options; keys still work. + assert _run_with_header('\x1b[B\r', 'tool_x\n{"a": 1}') == 1 diff --git a/tests/tui/test_tool_view.py b/tests/tui/test_tool_view.py new file mode 100644 index 000000000..06aa9fc2a --- /dev/null +++ b/tests/tui/test_tool_view.py @@ -0,0 +1,59 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Compact tool presentation: action headers + result summaries.""" +from ms_agent.tui.tool_view import tool_header, tool_summary + + +def test_write_file_header(): + assert tool_header('file_system---write_file', + {'path': '/a/b.md', 'content': 'x'}) == 'Write /a/b.md' + + +def test_shell_header(): + assert tool_header('code_executor---shell_executor', + {'command': 'git push'}) == 'Run git push' + + +def test_search_header(): + assert tool_header('web_search---exa_search', + {'query': 'modelscope'}) == 'Search modelscope' + + +def test_read_header_from_json_string_args(): + assert tool_header('file_system---read_file', + '{"path": "x.py"}') == 'Read x.py' + + +def test_generic_header_falls_back_to_action_and_arg(): + h = tool_header('custom---do_thing', {'foo': 'bar'}) + assert h == 'do thing bar' + + +def test_header_without_splitter(): + assert tool_header('web_search', {'query': 'q'}) == 'Search q' + + +def test_summary_empty_is_no_output(): + assert tool_summary('') == '(no output)' + + +def test_summary_single_line(): + assert tool_summary('wrote file') == 'wrote file' + + +def test_summary_multiline_counts_extra(): + s = tool_summary('line1\nline2\nline3') + assert 'line1' in s and '+2 lines' in s + + +def test_summary_trivial_first_line_leads_with_count(): + # JSON blob whose first line is just "{" → "58 lines", not "{ (+57 lines)". + blob = '{\n' + '\n'.join(f' "k{i}": {i}' for i in range(56)) + '\n}' + assert tool_summary(blob) == f'{blob.count(chr(10)) + 1} lines' + + +def test_summary_error(): + assert tool_summary('', 'boom') == 'error: boom' + + +def test_summary_truncates_long(): + assert tool_summary('x' * 200).endswith('…') diff --git a/tests/tui/test_tui_input.py b/tests/tui/test_tui_input.py new file mode 100644 index 000000000..589378d68 --- /dev/null +++ b/tests/tui/test_tui_input.py @@ -0,0 +1,59 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""PromptToolkitInput: status bar formatting + non-tty fallback.""" +import asyncio +from unittest.mock import patch + +from ms_agent.command import CommandRouter, register_builtin_commands +from ms_agent.tui.input import PromptToolkitInput, slash_matches +from ms_agent.tui.state import TuiState +from ms_agent.ui.input import InputSource + + +def _router(): + r = CommandRouter() + register_builtin_commands(r) + return r + + +def test_slash_matches_prefix(): + names = [n for n, _ in slash_matches(_router(), '/mod')] + assert 'model' in names + + +def test_slash_matches_bare_slash_lists_all(): + names = [n for n, _ in slash_matches(_router(), '/')] + assert 'help' in names and 'model' in names + + +def test_slash_no_match_for_plain_text(): + assert list(slash_matches(_router(), 'hello world')) == [] + + +def test_slash_completion_carries_description(): + desc = dict(slash_matches(_router(), '/model')).get('model') + assert desc # non-empty description shown as menu meta + + +def test_satisfies_input_source_protocol(): + assert isinstance(PromptToolkitInput(TuiState()), InputSource) + + +def test_toolbar_shows_state(): + state = TuiState(model='qwen3.7-plus', perm='restricted', + session_name='my-session') + state.total_prompt_tokens = 100 + state.total_completion_tokens = 20 + tb = PromptToolkitInput(state)._toolbar() + assert 'qwen3.7-plus' in tb + assert '120 tok' in tb + assert 'restricted' in tb + assert 'my-session' in tb + + +def test_read_prompt_falls_back_when_not_tty(): + inp = PromptToolkitInput(TuiState()) + with patch('ms_agent.tui.input.sys.stdin') as stdin, \ + patch('builtins.input', return_value='hello'): + stdin.isatty.return_value = False + result = asyncio.run(inp.read_prompt('> ')) + assert result == 'hello' diff --git a/tests/ui/test_agent_seam.py b/tests/ui/test_agent_seam.py new file mode 100644 index 000000000..a6e4917ce --- /dev/null +++ b/tests/ui/test_agent_seam.py @@ -0,0 +1,62 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""LLMAgent output seam: emit helpers route to the event sink or stdout. + +Pins the seam contract: with an event sink the agent emits semantic +AgentEvents; with no sink the code path is byte-identical to the pre-change +stdout behavior (CLI zero-regression). +""" +from omegaconf import OmegaConf + +from ms_agent.agent.llm_agent import LLMAgent +from ms_agent.ui.events import (ContentDelta, ContentEnd, ReasoningDelta, + ReasoningEnded, ReasoningStarted, RecordingSink) + + +def _agent(event_sink=None, reasoning_output='stdout'): + """Partially construct an agent (bypass __init__) with just the seam deps.""" + a = LLMAgent.__new__(LLMAgent) + a._event_sink = event_sink + a.config = OmegaConf.create( + {'generation_config': {'reasoning_output': reasoning_output}}) + return a + + +# ── sink path: structured events ────────────────────────────────────────── + + +def test_sink_receives_content_events(): + sink = RecordingSink() + a = _agent(event_sink=sink) + a._emit_content('hel') + a._emit_content('lo') + a._emit_content_end() + assert sink.events == [ContentDelta('hel'), ContentDelta('lo'), ContentEnd()] + + +def test_sink_receives_reasoning_events(): + sink = RecordingSink() + a = _agent(event_sink=sink) + a._emit_reasoning_start() + a._emit_reasoning_delta('thinking...') + a._emit_reasoning_end() + assert sink.events == [ + ReasoningStarted(), ReasoningDelta('thinking...'), ReasoningEnded()] + + +# ── no-sink path: byte-identical to current CLI behavior ────────────────── + + +def test_no_sink_content_goes_to_stdout(capsys): + a = _agent() # no sink -> stdout + a._emit_content('hello') + a._emit_content_end() + assert capsys.readouterr().out == 'hello\n' # exact legacy output + + +def test_no_sink_reasoning_goes_to_stdout(capsys): + a = _agent(reasoning_output='stdout') + a._emit_reasoning_start() + a._emit_reasoning_delta('mulling') + a._emit_reasoning_end() + out = capsys.readouterr().out + assert 'thinking' in out and 'mulling' in out diff --git a/tests/ui/test_events.py b/tests/ui/test_events.py new file mode 100644 index 000000000..6c6f11954 --- /dev/null +++ b/tests/ui/test_events.py @@ -0,0 +1,75 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Event model contract: serialization, discriminators, RecordingSink.""" +from ms_agent.ui.events import (AgentEventSink, ContentDelta, ContentEnd, + ContextCompacted, ErrorRaised, Notice, + PermissionRequested, PlanEntry, PlanUpdated, + ReasoningDelta, RecordingSink, ToolCallCompleted, + ToolCallStarted, TurnCompleted, UsageInfo) + + +def test_content_delta_to_dict(): + assert ContentDelta('hi').to_dict() == {'type': 'content_delta', 'text': 'hi'} + + +def test_type_property_matches_dict(): + ev = ToolCallStarted(call_id='c1', name='read_file', arguments={'p': 'x'}) + assert ev.type == 'tool_call_started' + assert ev.to_dict()['type'] == 'tool_call_started' + assert ev.to_dict()['arguments'] == {'p': 'x'} + + +def test_frozen_events_are_immutable(): + ev = ContentDelta('x') + try: + ev.text = 'y' # type: ignore[misc] + except Exception as e: + assert 'FrozenInstanceError' in type(e).__name__ or isinstance( + e, AttributeError) + else: + raise AssertionError('event should be immutable') + + +def test_nested_usage_serializes(): + ev = TurnCompleted( + turn_id='t1', + usage=UsageInfo(prompt_tokens=10, completion_tokens=5)) + d = ev.to_dict() + assert d['type'] == 'turn_completed' + assert d['usage'] == { + 'prompt_tokens': 10, 'completion_tokens': 5, 'reasoning_tokens': 0, + 'total_prompt_tokens': 0, 'total_completion_tokens': 0} + + +def test_nested_plan_entries_serialize(): + ev = PlanUpdated(entries=[PlanEntry('step 1', 'completed'), + PlanEntry('step 2')]) + d = ev.to_dict() + assert d['type'] == 'plan_updated' + assert d['entries'] == [ + {'content': 'step 1', 'status': 'completed'}, + {'content': 'step 2', 'status': 'pending'}] + + +def test_discriminators_are_unique_and_stable(): + events = [ContentDelta(), ContentEnd(), ReasoningDelta(), + ToolCallStarted(), ToolCallCompleted(), PlanUpdated(), + PermissionRequested(), ContextCompacted(), Notice(), + ErrorRaised(), TurnCompleted()] + types = [e.type for e in events] + assert len(types) == len(set(types)), 'event discriminators must be unique' + # Stable wire names a WebUI front-end mirrors — guard against renames. + assert ContentDelta().type == 'content_delta' + assert ToolCallCompleted().type == 'tool_call_completed' + assert PermissionRequested().type == 'permission_requested' + + +def test_recording_sink_collects_and_helpers(): + sink = RecordingSink() + assert isinstance(sink, AgentEventSink) # runtime_checkable protocol + sink.emit(ContentDelta('hello ')) + sink.emit(ToolCallStarted(call_id='1', name='shell')) + sink.emit(ContentDelta('world')) + assert sink.types() == ['content_delta', 'tool_call_started', 'content_delta'] + assert sink.text() == 'hello world' + assert len(sink.of_type('content_delta')) == 2 + assert sink.of_type('tool_call_started')[0].name == 'shell' diff --git a/tests/ui/test_input.py b/tests/ui/test_input.py new file mode 100644 index 000000000..cbfe8c9ff --- /dev/null +++ b/tests/ui/test_input.py @@ -0,0 +1,41 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""Async input contract: StdinInputSource reads a line off the event loop.""" +import asyncio +from unittest.mock import patch + +from ms_agent.ui.input import InputSource, StdinInputSource + + +def test_stdin_input_source_satisfies_protocol(): + assert isinstance(StdinInputSource(), InputSource) + + +def test_stdin_input_source_reads_line(): + src = StdinInputSource() + with patch('builtins.input', return_value='hello world'): + result = asyncio.run(src.read_prompt('>>> ')) + assert result == 'hello world' + + +def test_stdin_input_source_passes_prompt(): + src = StdinInputSource() + seen = {} + + def _fake_input(prompt): + seen['prompt'] = prompt + return 'x' + + with patch('builtins.input', _fake_input): + asyncio.run(src.read_prompt('agent> ')) + assert seen['prompt'] == 'agent> ' + + +def test_stdin_input_source_propagates_eof(): + src = StdinInputSource() + with patch('builtins.input', side_effect=EOFError): + try: + asyncio.run(src.read_prompt()) + except EOFError: + pass + else: + raise AssertionError('EOFError should propagate for quit handling') diff --git a/tests/ui/test_plan_extract.py b/tests/ui/test_plan_extract.py new file mode 100644 index 000000000..8f3300c19 --- /dev/null +++ b/tests/ui/test_plan_extract.py @@ -0,0 +1,44 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""LLMAgent._extract_plan_from_tool_result → PlanUpdated entries (todo panel).""" +from ms_agent.agent.llm_agent import LLMAgent +from ms_agent.llm.utils import Message + + +def _tool(name, content): + return Message(role='tool', name=name, content=content) + + +def test_todo_write_result_becomes_entries(): + m = _tool('todo_write', + '{"todos": [{"content": "step 1", "status": "in_progress"},' + ' {"content": "step 2"}]}') + entries = LLMAgent._extract_plan_from_tool_result(m) + assert len(entries) == 2 + assert entries[0].content == 'step 1' and entries[0].status == 'in_progress' + assert entries[1].content == 'step 2' and entries[1].status == 'pending' + + +def test_server_prefixed_todo_name_matches(): + m = _tool('todolist---todo_read', '[{"task": "a", "status": "completed"}]') + entries = LLMAgent._extract_plan_from_tool_result(m) + assert len(entries) == 1 and entries[0].content == 'a' + + +def test_split_task_matches(): + assert LLMAgent._extract_plan_from_tool_result( + _tool('split_task', '{"todos": [{"content": "x"}]}')) is not None + + +def test_non_todo_tool_ignored(): + assert LLMAgent._extract_plan_from_tool_result( + _tool('file_system---read_file', 'contents')) is None + + +def test_malformed_json_ignored(): + assert LLMAgent._extract_plan_from_tool_result( + _tool('todo_write', 'not json')) is None + + +def test_empty_todos_returns_none(): + assert LLMAgent._extract_plan_from_tool_result( + _tool('todo_write', '{"todos": []}')) is None diff --git a/tests/ui/test_webui_contract.py b/tests/ui/test_webui_contract.py new file mode 100644 index 000000000..929e42b76 --- /dev/null +++ b/tests/ui/test_webui_contract.py @@ -0,0 +1,87 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""WebUI-facing contract validation. + +The TUI is the first consumer of the ``ms_agent.ui`` seams; a WebUI backend is +the second. These tests pin the two pieces a WebUI relies on that the TUI +doesn't itself exercise via rendering: the serialized event stream (what gets +forwarded over a socket) and the async permission round-trip. +""" +import asyncio +import json + +from ms_agent.permission.handler import (PermissionAction, PermissionResponse, + WebPermissionHandler) +from ms_agent.ui.events import (ContentDelta, JsonlEventSink, + PermissionRequested, RecordingSink, + TeeEventSink, ToolCallStarted) + + +# ── event fan-out + serialization ───────────────────────────────────────── + + +def test_tee_fans_out_to_all_sinks(): + a, b = RecordingSink(), RecordingSink() + tee = TeeEventSink(a, b, None) # None sink is skipped + tee.emit(ContentDelta('hi')) + assert a.text() == 'hi' and b.text() == 'hi' + + +def test_jsonl_sink_writes_wire_payloads(tmp_path): + path = tmp_path / 'events.jsonl' + sink = JsonlEventSink(str(path)) + sink.emit(ContentDelta('hello')) + sink.emit(ToolCallStarted(call_id='c1', name='shell', arguments={'cmd': 'ls'})) + sink.close() + lines = [json.loads(l) for l in path.read_text().splitlines()] + assert lines[0] == {'type': 'content_delta', 'text': 'hello'} + assert lines[1]['type'] == 'tool_call_started' + assert lines[1]['call_id'] == 'c1' + assert lines[1]['arguments'] == {'cmd': 'ls'} + + +# ── async permission round-trip (WebPermissionHandler) ──────────────────── + + +class _Emitter: + def __init__(self): + self.events = [] + + def emit(self, event: dict) -> None: + self.events.append(event) + + +def test_web_permission_roundtrip(): + async def run(): + em = _Emitter() + handler = WebPermissionHandler(em, timeout=5.0) + task = asyncio.create_task( + handler.ask('web_search---exa', {'q': 'x'}, 'needs approval')) + await asyncio.sleep(0.01) # let ask() emit the request + + assert len(em.events) == 1 + ev = em.events[0] + assert ev['type'] == 'permission_request' + assert ev['tool_name'] == 'web_search---exa' + # The emitted dict must carry every field the PermissionRequested event + # models, so a WebUI can render from either representation. + for field in ('request_id', 'tool_name', 'tool_args', 'context', + 'options'): + assert field in ev + assert set(PermissionRequested().to_dict()) - {'type'} <= set(ev) + + handler.resolve( + ev['request_id'], + PermissionResponse(action=PermissionAction.ALLOW_ONCE)) + resp = await task + assert resp.action == PermissionAction.ALLOW_ONCE + + asyncio.run(run()) + + +def test_web_permission_times_out_to_deny(): + async def run(): + handler = WebPermissionHandler(_Emitter(), timeout=0.05) + resp = await handler.ask('shell', {'cmd': 'rm'}, '') + assert resp.action == PermissionAction.DENY + + asyncio.run(run()) diff --git a/tests/utils/test_logger.py b/tests/utils/test_logger.py new file mode 100644 index 000000000..b2eaa96da --- /dev/null +++ b/tests/utils/test_logger.py @@ -0,0 +1,26 @@ +# Copyright (c) ModelScope Contributors. All rights reserved. +"""_resolve_log_file must never crash logging setup (and thus the app) on a +filesystem error when opt-in file logging is enabled.""" +from unittest.mock import MagicMock, patch + +from ms_agent.utils.logger import _resolve_log_file + + +def test_default_console_only(monkeypatch): + monkeypatch.delenv('MS_AGENT_LOG_FILE', raising=False) + monkeypatch.delenv('LOG_FILE', raising=False) + assert _resolve_log_file(None) is None + + +def test_explicit_arg_wins(): + assert _resolve_log_file('/tmp/custom.log') == '/tmp/custom.log' + + +def test_mkdir_failure_falls_back_to_console(monkeypatch): + monkeypatch.setenv('MS_AGENT_LOG_FILE', 'true') + fake_dir = MagicMock() + fake_dir.mkdir.side_effect = OSError('read-only filesystem') + with patch( + 'ms_agent.project.paths.global_logs_dir', return_value=fake_dir): + # Must not raise; degrades to console-only (None). + assert _resolve_log_file(None) is None