From 79c1175bca38883de8173addbb13492bbba004ea Mon Sep 17 00:00:00 2001 From: waynehao Date: Tue, 15 Sep 2026 17:44:05 +0800 Subject: [PATCH] feat(agent-observability): add local observability dashboard for devflow runs Adds a Classic-only, opt-in skill that hooks into SessionStart / UserPromptSubmit / PreToolUse / PostToolUse / Stop to record tool calls, token usage, and cost into a local metrics log, plus a script that aggregates it (together with devflow's workflow-state.json) into a static dashboard: per-day cost/tokens, session and tool-call timelines, devflow stage progress, Skill/Rule hit rates, and Auto-Dispatch health. Also includes: - custom per-model pricing overrides for models missing from the built-in price table - accuracy fixes for the dashboard's turn/duration aggregation, dispatch attribution, session counting, and devflow stage classification (medium/large runs no longer misread as SOLO) - a fix for build-classic-hosts.py's generated-bundle check incorrectly flagging __pycache__/.pyc as unmanaged extras - cost-by-model chart fixes: long model names no longer get cut off by their bar, the chart gets a fairer share of its row's width, and the Token composition chart next to it is stretched to fill its row's actual height instead of leaving a gap underneath Co-Authored-By: Claude Sonnet 5 --- .claude/skills/agent-observability/SKILL.md | 33 + .../agent-observability/config/pricing.json | 39 + .../skills/agent-observability/logs/.gitkeep | 0 .../references/quickstart.md | 195 +++ .../references/schema-v2.md | 282 ++++ .../agent-observability/scripts/__init__.py | 1 + .../scripts/build_dashboard_data.py | 624 +++++++++ .../scripts/core/__init__.py | 8 + .../scripts/core/agent_identity.py | 587 ++++++++ .../scripts/core/agentlens/__init__.py | 9 + .../scripts/core/agentlens/bootstrap.py | 229 +++ .../scripts/core/agentlens/normalize.py | 323 +++++ .../scripts/core/agentlens/runtime.py | 761 ++++++++++ .../scripts/core/agentlens/tracing.py | 504 +++++++ .../scripts/core/cls_sink.py | 470 +++++++ .../scripts/core/cls_uploader.mjs | 96 ++ .../scripts/core/collector.py | 1008 ++++++++++++++ .../scripts/core/devflow.py | 234 ++++ .../scripts/core/emitter.py | 578 ++++++++ .../scripts/core/runtime.py | 598 ++++++++ .../scripts/core/scanner.py | 496 +++++++ .../agent-observability/scripts/core/state.py | 1222 +++++++++++++++++ .../scripts/dashboard/index.html | 1086 +++++++++++++++ .../agent-observability/scripts/main.py | 23 + .../agent-observability/scripts/run_hook.sh | 16 + .../templates/settings-hook.json | 62 + .../tests/fixtures/transcript.jsonl | 10 + .../tests/test_agent_identity.py | 84 ++ .../tests/test_build_dashboard_data.py | 603 ++++++++ .../tests/test_cls_sink.py | 184 +++ .../tests/test_collector.py | 715 ++++++++++ .../agent-observability/tests/test_devflow.py | 389 ++++++ .../tests/test_pricing_overrides.py | 394 ++++++ .../tests/test_runtime_flow.py | 74 + .codebuddy/runtime/workflow-state-spec.md | 2 + .codebuddy/settings.json | 57 +- .../skills/agent-observability/SKILL.md | 33 + .../agent-observability/config/pricing.json | 39 + .../skills/agent-observability/logs/.gitkeep | 0 .../references/quickstart.md | 195 +++ .../references/schema-v2.md | 282 ++++ .../agent-observability/scripts/__init__.py | 1 + .../scripts/build_dashboard_data.py | 624 +++++++++ .../scripts/core/__init__.py | 8 + .../scripts/core/agent_identity.py | 587 ++++++++ .../scripts/core/agentlens/__init__.py | 9 + .../scripts/core/agentlens/bootstrap.py | 229 +++ .../scripts/core/agentlens/normalize.py | 323 +++++ .../scripts/core/agentlens/runtime.py | 761 ++++++++++ .../scripts/core/agentlens/tracing.py | 504 +++++++ .../scripts/core/cls_sink.py | 470 +++++++ .../scripts/core/cls_uploader.mjs | 96 ++ .../scripts/core/collector.py | 1008 ++++++++++++++ .../scripts/core/devflow.py | 234 ++++ .../scripts/core/emitter.py | 578 ++++++++ .../scripts/core/runtime.py | 598 ++++++++ .../scripts/core/scanner.py | 496 +++++++ .../agent-observability/scripts/core/state.py | 1222 +++++++++++++++++ .../scripts/dashboard/index.html | 1086 +++++++++++++++ .../agent-observability/scripts/main.py | 23 + .../agent-observability/scripts/run_hook.sh | 16 + .../templates/settings-hook.json | 62 + .../tests/fixtures/transcript.jsonl | 10 + .../tests/test_agent_identity.py | 84 ++ .../tests/test_build_dashboard_data.py | 603 ++++++++ .../tests/test_cls_sink.py | 184 +++ .../tests/test_collector.py | 715 ++++++++++ .../agent-observability/tests/test_devflow.py | 389 ++++++ .../tests/test_pricing_overrides.py | 394 ++++++ .../tests/test_runtime_flow.py | 74 + .cursor/skills/agent-observability/SKILL.md | 33 + .../agent-observability/config/pricing.json | 39 + .../skills/agent-observability/logs/.gitkeep | 0 .../references/quickstart.md | 195 +++ .../references/schema-v2.md | 282 ++++ .../agent-observability/scripts/__init__.py | 1 + .../scripts/build_dashboard_data.py | 624 +++++++++ .../scripts/core/__init__.py | 8 + .../scripts/core/agent_identity.py | 587 ++++++++ .../scripts/core/agentlens/__init__.py | 9 + .../scripts/core/agentlens/bootstrap.py | 229 +++ .../scripts/core/agentlens/normalize.py | 323 +++++ .../scripts/core/agentlens/runtime.py | 761 ++++++++++ .../scripts/core/agentlens/tracing.py | 504 +++++++ .../scripts/core/cls_sink.py | 470 +++++++ .../scripts/core/cls_uploader.mjs | 96 ++ .../scripts/core/collector.py | 1008 ++++++++++++++ .../scripts/core/devflow.py | 234 ++++ .../scripts/core/emitter.py | 578 ++++++++ .../scripts/core/runtime.py | 598 ++++++++ .../scripts/core/scanner.py | 496 +++++++ .../agent-observability/scripts/core/state.py | 1222 +++++++++++++++++ .../scripts/dashboard/index.html | 1086 +++++++++++++++ .../agent-observability/scripts/main.py | 23 + .../agent-observability/scripts/run_hook.sh | 16 + .../templates/settings-hook.json | 62 + .../tests/fixtures/transcript.jsonl | 10 + .../tests/test_agent_identity.py | 84 ++ .../tests/test_build_dashboard_data.py | 603 ++++++++ .../tests/test_cls_sink.py | 184 +++ .../tests/test_collector.py | 715 ++++++++++ .../agent-observability/tests/test_devflow.py | 389 ++++++ .../tests/test_pricing_overrides.py | 394 ++++++ .../tests/test_runtime_flow.py | 74 + .gitignore | 8 + scripts/build-classic-hosts.py | 8 +- 106 files changed, 35884 insertions(+), 2 deletions(-) create mode 100644 .claude/skills/agent-observability/SKILL.md create mode 100644 .claude/skills/agent-observability/config/pricing.json create mode 100644 .claude/skills/agent-observability/logs/.gitkeep create mode 100644 .claude/skills/agent-observability/references/quickstart.md create mode 100644 .claude/skills/agent-observability/references/schema-v2.md create mode 100644 .claude/skills/agent-observability/scripts/__init__.py create mode 100644 .claude/skills/agent-observability/scripts/build_dashboard_data.py create mode 100644 .claude/skills/agent-observability/scripts/core/__init__.py create mode 100644 .claude/skills/agent-observability/scripts/core/agent_identity.py create mode 100644 .claude/skills/agent-observability/scripts/core/agentlens/__init__.py create mode 100644 .claude/skills/agent-observability/scripts/core/agentlens/bootstrap.py create mode 100644 .claude/skills/agent-observability/scripts/core/agentlens/normalize.py create mode 100644 .claude/skills/agent-observability/scripts/core/agentlens/runtime.py create mode 100644 .claude/skills/agent-observability/scripts/core/agentlens/tracing.py create mode 100644 .claude/skills/agent-observability/scripts/core/cls_sink.py create mode 100644 .claude/skills/agent-observability/scripts/core/cls_uploader.mjs create mode 100644 .claude/skills/agent-observability/scripts/core/collector.py create mode 100644 .claude/skills/agent-observability/scripts/core/devflow.py create mode 100644 .claude/skills/agent-observability/scripts/core/emitter.py create mode 100644 .claude/skills/agent-observability/scripts/core/runtime.py create mode 100644 .claude/skills/agent-observability/scripts/core/scanner.py create mode 100644 .claude/skills/agent-observability/scripts/core/state.py create mode 100644 .claude/skills/agent-observability/scripts/dashboard/index.html create mode 100644 .claude/skills/agent-observability/scripts/main.py create mode 100644 .claude/skills/agent-observability/scripts/run_hook.sh create mode 100644 .claude/skills/agent-observability/templates/settings-hook.json create mode 100644 .claude/skills/agent-observability/tests/fixtures/transcript.jsonl create mode 100644 .claude/skills/agent-observability/tests/test_agent_identity.py create mode 100644 .claude/skills/agent-observability/tests/test_build_dashboard_data.py create mode 100644 .claude/skills/agent-observability/tests/test_cls_sink.py create mode 100644 .claude/skills/agent-observability/tests/test_collector.py create mode 100644 .claude/skills/agent-observability/tests/test_devflow.py create mode 100644 .claude/skills/agent-observability/tests/test_pricing_overrides.py create mode 100644 .claude/skills/agent-observability/tests/test_runtime_flow.py create mode 100644 .codebuddy/skills/agent-observability/SKILL.md create mode 100644 .codebuddy/skills/agent-observability/config/pricing.json create mode 100644 .codebuddy/skills/agent-observability/logs/.gitkeep create mode 100644 .codebuddy/skills/agent-observability/references/quickstart.md create mode 100644 .codebuddy/skills/agent-observability/references/schema-v2.md create mode 100644 .codebuddy/skills/agent-observability/scripts/__init__.py create mode 100644 .codebuddy/skills/agent-observability/scripts/build_dashboard_data.py create mode 100644 .codebuddy/skills/agent-observability/scripts/core/__init__.py create mode 100644 .codebuddy/skills/agent-observability/scripts/core/agent_identity.py create mode 100644 .codebuddy/skills/agent-observability/scripts/core/agentlens/__init__.py create mode 100644 .codebuddy/skills/agent-observability/scripts/core/agentlens/bootstrap.py create mode 100644 .codebuddy/skills/agent-observability/scripts/core/agentlens/normalize.py create mode 100644 .codebuddy/skills/agent-observability/scripts/core/agentlens/runtime.py create mode 100644 .codebuddy/skills/agent-observability/scripts/core/agentlens/tracing.py create mode 100644 .codebuddy/skills/agent-observability/scripts/core/cls_sink.py create mode 100644 .codebuddy/skills/agent-observability/scripts/core/cls_uploader.mjs create mode 100644 .codebuddy/skills/agent-observability/scripts/core/collector.py create mode 100644 .codebuddy/skills/agent-observability/scripts/core/devflow.py create mode 100644 .codebuddy/skills/agent-observability/scripts/core/emitter.py create mode 100644 .codebuddy/skills/agent-observability/scripts/core/runtime.py create mode 100644 .codebuddy/skills/agent-observability/scripts/core/scanner.py create mode 100644 .codebuddy/skills/agent-observability/scripts/core/state.py create mode 100644 .codebuddy/skills/agent-observability/scripts/dashboard/index.html create mode 100644 .codebuddy/skills/agent-observability/scripts/main.py create mode 100755 .codebuddy/skills/agent-observability/scripts/run_hook.sh create mode 100644 .codebuddy/skills/agent-observability/templates/settings-hook.json create mode 100644 .codebuddy/skills/agent-observability/tests/fixtures/transcript.jsonl create mode 100644 .codebuddy/skills/agent-observability/tests/test_agent_identity.py create mode 100644 .codebuddy/skills/agent-observability/tests/test_build_dashboard_data.py create mode 100644 .codebuddy/skills/agent-observability/tests/test_cls_sink.py create mode 100644 .codebuddy/skills/agent-observability/tests/test_collector.py create mode 100644 .codebuddy/skills/agent-observability/tests/test_devflow.py create mode 100644 .codebuddy/skills/agent-observability/tests/test_pricing_overrides.py create mode 100644 .codebuddy/skills/agent-observability/tests/test_runtime_flow.py create mode 100644 .cursor/skills/agent-observability/SKILL.md create mode 100644 .cursor/skills/agent-observability/config/pricing.json create mode 100644 .cursor/skills/agent-observability/logs/.gitkeep create mode 100644 .cursor/skills/agent-observability/references/quickstart.md create mode 100644 .cursor/skills/agent-observability/references/schema-v2.md create mode 100644 .cursor/skills/agent-observability/scripts/__init__.py create mode 100644 .cursor/skills/agent-observability/scripts/build_dashboard_data.py create mode 100644 .cursor/skills/agent-observability/scripts/core/__init__.py create mode 100644 .cursor/skills/agent-observability/scripts/core/agent_identity.py create mode 100644 .cursor/skills/agent-observability/scripts/core/agentlens/__init__.py create mode 100644 .cursor/skills/agent-observability/scripts/core/agentlens/bootstrap.py create mode 100644 .cursor/skills/agent-observability/scripts/core/agentlens/normalize.py create mode 100644 .cursor/skills/agent-observability/scripts/core/agentlens/runtime.py create mode 100644 .cursor/skills/agent-observability/scripts/core/agentlens/tracing.py create mode 100644 .cursor/skills/agent-observability/scripts/core/cls_sink.py create mode 100644 .cursor/skills/agent-observability/scripts/core/cls_uploader.mjs create mode 100644 .cursor/skills/agent-observability/scripts/core/collector.py create mode 100644 .cursor/skills/agent-observability/scripts/core/devflow.py create mode 100644 .cursor/skills/agent-observability/scripts/core/emitter.py create mode 100644 .cursor/skills/agent-observability/scripts/core/runtime.py create mode 100644 .cursor/skills/agent-observability/scripts/core/scanner.py create mode 100644 .cursor/skills/agent-observability/scripts/core/state.py create mode 100644 .cursor/skills/agent-observability/scripts/dashboard/index.html create mode 100644 .cursor/skills/agent-observability/scripts/main.py create mode 100644 .cursor/skills/agent-observability/scripts/run_hook.sh create mode 100644 .cursor/skills/agent-observability/templates/settings-hook.json create mode 100644 .cursor/skills/agent-observability/tests/fixtures/transcript.jsonl create mode 100644 .cursor/skills/agent-observability/tests/test_agent_identity.py create mode 100644 .cursor/skills/agent-observability/tests/test_build_dashboard_data.py create mode 100644 .cursor/skills/agent-observability/tests/test_cls_sink.py create mode 100644 .cursor/skills/agent-observability/tests/test_collector.py create mode 100644 .cursor/skills/agent-observability/tests/test_devflow.py create mode 100644 .cursor/skills/agent-observability/tests/test_pricing_overrides.py create mode 100644 .cursor/skills/agent-observability/tests/test_runtime_flow.py diff --git a/.claude/skills/agent-observability/SKILL.md b/.claude/skills/agent-observability/SKILL.md new file mode 100644 index 0000000..8bff9f0 --- /dev/null +++ b/.claude/skills/agent-observability/SKILL.md @@ -0,0 +1,33 @@ +--- +name: agent-observability +description: Add or maintain CodeBuddy hook observability for projects that need local metrics for tool latency, transcript token usage, session cost, and multi-agent trace attribution. Use when wiring hook-based telemetry, debugging missing usage events, and validating AgentLens state. +--- + +## Workflow + +1. Merge `templates/settings-hook.json` into the user or project `settings.json`. +2. Keep hook entries command-only and keep timeouts around 5 to 10 seconds. +3. Run a normal CodeBuddy session and inspect `logs/metrics.ndjson`. +4. Inspect `logs/.state.json` when usage replay, offsets, or AgentLens turn state look wrong. +5. Keep the runtime generic and verify output from `metrics.ndjson` and `.state.json`. + +## Read Next + +- Read `references/quickstart.md` for setup and smoke-test steps. +- Read `references/schema-v2.md` when you need field definitions or sidecar state shape. +## Key Files + +- `scripts/main.py`: hook entrypoint for `session-start`, `user-prompt-submit`, `pre`, `post`, and `stop` +- `scripts/run_hook.sh`: stable shell wrapper for CodeBuddy hook commands +- `scripts/core/collector.py`: transcript parsing, pre/post pairing, and session usage recording +- `scripts/core/agentlens.py`: AgentLens trace emission and agent or step grouping +- `scripts/core/state.py`: persisted hook state, offsets, and AgentLens sidecar state +- `scripts/core/devflow.py`: optional devflow-awareness — detects a `multi-agents-devflow-*` team, reads `workflow-state.json`, and emits `stage_transition` events. No-ops entirely on non-devflow projects. +- `templates/settings-hook.json`: hook wiring template + +## Output Contract + +- Always emit `event`, `sid`, and `ts`. +- Emit `tool` for per-call latency and `usage` for transcript-derived token and cost events. +- Emit `stop` for end-of-session flush. +- Never block the main hook flow; hook exits must stay `0`. diff --git a/.claude/skills/agent-observability/config/pricing.json b/.claude/skills/agent-observability/config/pricing.json new file mode 100644 index 0000000..59d8e58 --- /dev/null +++ b/.claude/skills/agent-observability/config/pricing.json @@ -0,0 +1,39 @@ +{ + "claude-opus-4.8": {"input": 5.0, "output": 25.0, "cache_read": 0.5, "cache_write": 6.25}, + "claude-opus-4.7": {"input": 5.0, "output": 25.0, "cache_read": 0.5, "cache_write": 6.25}, + "claude-opus-4.6": {"input": 5.0, "output": 25.0, "cache_read": 0.5, "cache_write": 6.25}, + "claude-opus-4": {"input": 5.0, "output": 25.0, "cache_read": 0.5, "cache_write": 6.25}, + "claude-sonnet-4.6": {"input": 3.0, "output": 15.0, "cache_read": 0.3, "cache_write": 3.75}, + "claude-sonnet-4.5": {"input": 3.0, "output": 15.0, "cache_read": 0.3, "cache_write": 3.75}, + "claude-sonnet-4": {"input": 3.0, "output": 15.0, "cache_read": 0.3, "cache_write": 3.75}, + "claude-3-7-sonnet": {"input": 3.0, "output": 15.0, "cache_read": 0.3, "cache_write": 3.75}, + "claude-3-5-sonnet": {"input": 3.0, "output": 15.0, "cache_read": 0.3, "cache_write": 3.75}, + "claude-haiku-4.5": {"input": 1.0, "output": 5.0, "cache_read": 0.1, "cache_write": 1.25}, + "claude-3-5-haiku": {"input": 1.0, "output": 5.0, "cache_read": 0.1, "cache_write": 1.25}, + "claude-haiku-4": {"input": 1.0, "output": 5.0, "cache_read": 0.1, "cache_write": 1.25}, + "gpt-5.5-fast": {"input": 12.5, "output": 75.0, "cache_read": 1.25, "cache_write": 12.5}, + "gpt-5.5": {"input": 5.0, "output": 30.0, "cache_read": 0.5, "cache_write": 5.0}, + "gpt-5.4-fast": {"input": 5.0, "output": 30.0, "cache_read": 0.5, "cache_write": 5.0}, + "gpt-5.4": {"input": 2.5, "output": 15.0, "cache_read": 0.25, "cache_write": 2.5}, + "gpt-5.3-codex": {"input": 1.75, "output": 14.0, "cache_read": 0.175, "cache_write": 1.75}, + "gpt-4o": {"input": 2.5, "output": 10.0, "cache_read": 1.25, "cache_write": 2.5}, + "gpt-4o-mini": {"input": 0.15, "output": 0.6, "cache_read": 0.075, "cache_write": 0.15}, + "gemini-3.5-flash": {"input": 1.5, "output": 9.0, "cache_read": 0.15, "cache_write": 1.5}, + "gemini-3.1-pro": {"input": 2.0, "output": 12.0, "cache_read": 0.2, "cache_write": 2.0}, + "gemini-3.1-flash-lite": {"input": 0.25, "output": 1.5, "cache_read": 0.025, "cache_write": 0.25}, + "gemini-3-flash": {"input": 0.5, "output": 3.0, "cache_read": 0.05, "cache_write": 0.5}, + "gemini-2": {"input": 0.5, "output": 3.0, "cache_read": 0.05, "cache_write": 0.5}, + "glm-5.1": {"input": 0.857, "output": 3.429, "cache_read": 0.186, "cache_write": 0.857}, + "glm-5v-turbo": {"input": 0.714, "output": 3.143, "cache_read": 0.171, "cache_write": 0.714}, + "glm-5-turbo": {"input": 0.714, "output": 3.143, "cache_read": 0.171, "cache_write": 0.714}, + "glm-5": {"input": 0.857, "output": 3.429, "cache_read": 0.186, "cache_write": 0.857}, + "kimi-k2.6": {"input": 0.929, "output": 3.857, "cache_read": 0.157, "cache_write": 0.929}, + "kimi-k2.5": {"input": 0.571, "output": 3.0, "cache_read": 0.1, "cache_write": 0.571}, + "kimi": {"input": 0.571, "output": 3.0, "cache_read": 0.1, "cache_write": 0.571}, + "deepseek-v4-pro": {"input": 0.429, "output": 0.857, "cache_read": 0.004, "cache_write": 0.429}, + "deepseek-v4-flash": {"input": 0.143, "output": 0.286, "cache_read": 0.003, "cache_write": 0.143}, + "deepseek-v4": {"input": 0.429, "output": 0.857, "cache_read": 0.004, "cache_write": 0.429}, + "deepseek": {"input": 0.143, "output": 0.286, "cache_read": 0.003, "cache_write": 0.143}, + "minimax-m2.7": {"input": 0.3, "output": 1.2, "cache_read": 0.06, "cache_write": 0.3}, + "minimax": {"input": 0.3, "output": 1.2, "cache_read": 0.06, "cache_write": 0.3} +} diff --git a/.claude/skills/agent-observability/logs/.gitkeep b/.claude/skills/agent-observability/logs/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/.claude/skills/agent-observability/references/quickstart.md b/.claude/skills/agent-observability/references/quickstart.md new file mode 100644 index 0000000..4a66777 --- /dev/null +++ b/.claude/skills/agent-observability/references/quickstart.md @@ -0,0 +1,195 @@ +## Quickstart + +### 安装 + +将 `templates/settings-hook.json` 合并到 `~/.claude/settings.json` 或 `/.claude/settings.json`。 + +必须启用 hooks：`SessionStart`、`UserPromptSubmit`、`PreToolUse`、`PostToolUse`、`Stop`。 + +### 查看输出 + +完成一次正常会话后检查： +- `.claude/skills/agent-observability/logs/metrics.ndjson` +- `.claude/skills/agent-observability/logs/.state.json` + +### 指定数据源路径（可选） + +`build_dashboard_data.py` 默认从 `/logs/metrics.ndjson` 与 `/logs/.state.json` 读取，**不传参数时行为完全不变**。 + +如需从其它位置读取，可用 `--metrics-path` / `--state-path` 覆盖： + +```bash +python3 scripts/build_dashboard_data.py \ + --metrics-path ~/Downloads/metrics.ndjson \ + --state-path ~/Downloads/.state.json \ + --out dashboard/dashboard-data.json +``` + +注意： + +- 两个参数都留空（或省略）时回退到 `/logs/` 下的默认文件。 +- 路径支持 `~` 展开（如上例的 `~/Downloads/...`）。 +- 输出 `dashboard-data.json` 的 `source.metrics_ndjson` / `source.state_json` 会**如实反映实际读取到的路径**，覆盖后自然指向你给定的文件，便于核对数据来源。 + +重点字段： +- `event=tool`：工具耗时、`skill`、`rule` +- `event=usage`：`tokens`、`model`、`cost_usd` +- `event=stop`：会话尾部 flush 与总成本 + +### 导出最慢的工具调用（可选） + +`--top-slow N` 会在生成 `dashboard-data.json` 的同时，额外在**终端**打印耗时最长的 N 次 `event=tool` 调用（工具名 + 耗时 ms，按耗时降序）。排查"哪次工具调用拖慢了会话"时不用再去翻原始 ndjson。 + +```bash +python3 scripts/build_dashboard_data.py --top-slow 5 +``` + +输出示例（排在 `wrote ...` 之后）： + +``` +top-slow 5 tool calls (by ms): + 1. Bash 4820ms + 2. Grep 1230ms + 3. Read 310ms +``` + +注意： + +- **默认不开启**：不传该参数时不会打印任何额外内容，原有输出一字不变。 +- **只读**：结果只打印到终端，不会写进 `dashboard-data.json`——输出结构与不开时完全一致，看板不受影响。 +- 只统计 `event=tool` 事件；`ms` 缺失或非数字时按 `0` 兜底，与看板其它统计口径一致。 +- `N` 大于实际 tool 事件数时取全部，不报错；没有任何 tool 事件时不打印该段。 + +### 自定义模型定价覆盖（可选） + +内置价格表 `config/pricing.json` 覆盖不到的模型（自部署模型、内部代号、刚发布还没来得及收录的模型）， +`cost_usd` 会算不出来。给这些模型补单价不用改内置文件——写一份**只含差异**的覆盖文件即可。 + +**放哪**：默认 `/.claude/agent-observability/pricing.overrides.json` +（`` 是 `.claude/` 所在的那一层，例如 `/Users/me/my-project`）。 + +刻意放在 `skills/` 树**之外**，原因有三： + +- `scripts/build-classic-hosts.py` 会整棵同步 `.claude/skills` 到 `.claude/` `.cursor/`， + 放树内会让每次改动都产生 `--check` drift，还会把你的单价复制进生成的宿主包； +- `config/pricing.json` 属于 skill 自带资产，后续更新会把它冲掉，覆盖文件不会； +- 它是纯数据文件，删除或改名即可停用，不需要重启任何东西（hook 每次都是新进程）。 + +**格式**：键是模型名（匹配时忽略大小写与首尾空格），值是要覆盖的字段。 +单位与 `config/pricing.json` 完全一致——**USD / 1M tokens**，可用字段只有四个： +`input`、`output`、`cache_read`、`cache_write`。 + +```json +{ + "my-model": { "output": 9.0 }, + "gpt-4o": { "output": 99.0 } +} +``` + +**合并是字段级的**：上面这份只改 `output`——`my-model` 的 `input`/`cache_read`/`cache_write` +沿用内置值（内置表里没有 `my-model` 时，未覆盖的字段按 0 计），`gpt-4o` 的 `input` 仍是内置的 `2.5`。 +可以新增内置表里不存在的模型，但**不能删除**内置的模型或字段（合并只能加不能减）。 + +**两个环境变量**（都支持 `~` 展开）： + +| 环境变量 | 作用 | 优先级 | +|---|---|---| +| `AOBS_PRICES_PATH` | **替换**整张基础表（不再读内置 `config/pricing.json`） | 低 | +| `AOBS_PRICING_OVERRIDES_PATH` | 在上面那张基础表之上**叠加**一份补丁，指向任意路径 | 高 | + +两者同时设置时，覆盖文件里的字段最终生效。 + +> ⚠️ **只对新事件生效，不回溯**：`cost_usd` 在 hook 期就写进了 `metrics.ndjson`， +> 补价之后只有**之后新产生**的 usage 事件按新价格计算，已经落盘的历史成本不会被重算。 + +**出问题会静默降级**：覆盖文件不存在、JSON 非法、顶层不是对象、字段值不是数字， +四种情况都自动退回纯内置价格表——不报错、不打印、hook 照常退出 0 +（否则一个手误的格式问题会让整条 hook 链路不可用）。所以"改了没生效"通常意味着文件没被读到， +先跑一次诊断： + +```bash +python3 - <<'PY' +import sys +sys.path.insert(0, "scripts") +from core import emitter +print("path :", emitter.resolve_overrides_path()) +print("loaded :", emitter.load_price_overrides()) +print("gpt-4o :", emitter.load_prices().get("gpt-4o")) +PY +``` + +`path` 是实际读取的位置（不是你以为的那个），`loaded` 为空说明文件没读到或全被判为坏字段。 + +**看板上的提示**：「建议关注」卡片第 4 条「模型定价」会列出**完全没命中价格表**的模型 +（模型名 + 缺少成本的 usage 事件数）。模糊匹配（最长子串）命中的模型能算出成本，不会出现在里面。 +处置方式就是把该模型写进覆盖文件；所有模型都有价时这条显示 green。 + +### 常见问题 + +- 没有日志：检查 hook 命令路径。 +- `ms` 为空：通常是 pre/post 未配对。 +- 没有 `usage`：`transcript_path` 缺失、tail 无 usage，或增量已去重。 +- AgentLens 未启用：检查 `.state.json` 的 `_agentlens.enabled` / `last_error`。 + +### 已知问题（未修复） + +- **工具失败事件的 `raw_response` 经常缺失（PostToolUse 与 transcript 落盘之间的时序竞争）**： + 真实 CodeBuddy CLI 场景下实测复现过——一次会故意制造失败的 `Bash` 调用（`exitCode=1`， + transcript 里 `function_call_result.providerData.toolResult.rawResponse` 确实带了 + `is_error:true`/`exitCode:1`/`tool_error_code`），但最终写进 `metrics.ndjson` 的 + `tool` 事件的 `tool_details` 里完全没有 `raw_response` 字段。 + 用实测时间戳定位到根因：PostToolUse hook 记录这次调用的时间是 `ts=...396.524`， + 但 transcript 里 `function_call_result` 真正落盘的时间是 `timestamp=...396.609`—— + **晚了 85ms**。hook 触发时去扫 transcript 文件的那一刻，CLI 还没来得及把执行结果 + 那条记录写盘，`find_current_tool_context()`（`core/collector.py`）只能看到 + `function_call`（请求），看不到 `function_call_result`（结果）。已经单独验证过 + `merge_tool_records()` / `tool_details_from_record()` 的合并逻辑本身没问题—— + 只要数据真的已经在文件里，能正确解析出完整 `raw_response`；问题纯粹是读的时机 + 比 CLI 写盘早了一步。`collect_transcript_entries()` 用的是持久化的增量字节 offset + 游标，错过这次窗口后不会在后续调用里回头补扫，所以这次机会永久丢失，直接后果是 + `build_dashboard_data.py` 的 `tool_call_failed()`（无论怎么改判定逻辑）都拿不到 + 数据，"工具失败率"/`failures` 列表对这类快速失败调用会漏检。 + 复现方式：让 CodeBuddy 执行一个必然快速失败的命令（如 `ls /path/does/not/exist`， + 越快的命令越容易复现，因为 hook 触发与 transcript 落盘之间的竞争窗口更紧张）， + 对比 `metrics.ndjson` 里该 `tool` 事件的 `tool_details.raw_response` 是否存在， + 和对应 transcript `.jsonl` 里 `function_call`/`function_call_result` 两条记录各自 + 的 `timestamp` 先后。 + + **影响范围（已精确定位，不是猜测）**：`raw_response` 在 `build_dashboard_data.py` + 里只有两处消费者——`tool_call_failed()`（喂给 `build_daily_and_sessions()` 的 + `day.failures` 计数和 `session.status`）和 `build_failures()`（"工具失败面板"的 + 数据源）。真实数据统计过：107 次 tool 事件里只有 2 次带 `raw_response`，且只有 + `Bash` 调用会带这个字段（Read/Edit/Write/SendMessage 等其它工具从不带，不受 + 此问题影响）。缺失时 `tool_call_failed()` 默认判"未失败"，所以效果是**恒定 + 漏报，不会误报**——工具失败率/失败面板/会话状态列显示的"正常"可能掩盖了真实 + 发生过的 Bash 失败。turns、duration、dispatch、cost、token、skill/rule 命中、 + devflow 阶段耗时、会话列表本身完全不受影响，是纯观测盲区，不影响 devflow 实际 + 执行行为。 + + **三次修复尝试均已失败，均已回滚（详见下方"已尝试且已放弃的修复方向"）**： + 真正阻塞方向 A 的证据很反常——三次独立测试里，预算从 160ms 加到 500ms 再加到 + 2000ms，实测缺口每次都精确地"比预算多几十到一百多毫秒"（206/564/2151ms）， + 不像是在等一个独立发生的固定延迟，更像是**hook 自己的同步等待在阻塞 CLI 落盘** + ——等得越久，结果来得越晚。这个因果关系还没验证清楚，在验证清楚之前，继续在 + hook 里加同步等待大概率是死路，不建议再尝试。 + +### 已尝试且已放弃的修复方向（供以后参考，避免重复踩坑） + +1. **方向 A：PostToolUse 里同步有界重试**（`claim=False` 轮询直到等到 `raw_response` + 或超时，只对 `tool_name=="Bash"` 生效）。三次真实端到端验证，预算 160ms/500ms/ + 2000ms 全部失败，且"缺口≈预算+常数"的规律强烈暗示等待本身可能在拖慢 CLI + 落盘（见上文）。不建议在搞清楚这层因果关系之前继续加大预算。 +2. **方向 B：推迟到 `pending_tool_emits` 重试队列，下次 hook 触发或 session Stop + 时再补**（`claim=False` 探测 + 延后 `claim=True`，避免过早消费掉 call_id）。 + 逻辑和单元测试都通过，但端到端验证暴露了一个更深的、独立于这次修复的既有 + 架构问题：`find_current_tool_context()`（工具上下文查询）和 + `emit_transcript_events()`（usage token 扫描）共用同一个持久化字节偏移游标 + （`core/state.py` 里只按 `(sid, transcript_path)` 区分，不分用途）。 + `emit_transcript_events()` 每次 `handle_post()` 都无条件推进这个游标，一旦 + 推过某段内容，后续任何工具上下文重试在这段范围内都会**彻底找不到任何数据** + （不只是缺 `raw_response`，连 `call_id`/`arguments` 都没了）——比不修复更糟。 + 真实验证过两次：两条端到端测试调用最终落盘时 `tool_details`完全是空的。 + 要让方向 B 真正可行，必须先给工具上下文查询一个独立于 usage 扫描的游标， + 这是范围更大、需要认真设计的改动，还没有细化方案。 + +以上两个方向的实现和回滚记录详见会话 memory（`project-agent-observability-raw-response-race`）。 diff --git a/.claude/skills/agent-observability/references/schema-v2.md b/.claude/skills/agent-observability/references/schema-v2.md new file mode 100644 index 0000000..43b84e1 --- /dev/null +++ b/.claude/skills/agent-observability/references/schema-v2.md @@ -0,0 +1,282 @@ +## Schema v2.3 + +`agent-observability` 输出 `ndjson`，每行一个事件对象。 + +> 以下字段表基于当前 `logs/metrics.ndjson` 的实际输出整理；其中“可选字段”只会在特定场景出现。 + +### 通用字段 + +- `event`: `start | user_prompt_submit | tool | usage | stop | error | stage_transition` +- `sid`: session id +- `ts`: 秒级时间戳（float） +- `turn_id`: 可选；由 `UserPromptSubmit` 生成，贯穿本轮事件 + +### Devflow 感知字段（可选，见 `core/devflow.py`） + +只有当前项目正在跑 `.claude/runtime` 描述的 multi-agents-devflow 工作流（存在 +`.claude/teams/multi-agents-devflow-{task_slug}/` team 目录）时才会出现，非 +devflow 项目完全不受影响： + +- `tool` / `usage` / `stop` 事件上会附带 `task_slug`（devflow 需求标识，用于把 + main + 最多 7 个常驻角色跨事件串成"同一次运行"；⚠️ `/resume-devflow` 中断恢复 + 可能发生在新的 CodeBuddy 顶层 session 里，因此**聚合一次 devflow 运行要按 + `task_slug` 分组，不能按 `sid` 分组**）与 `stage`（`workflow-state.json` 的 + `current_stage`，如 `TASK-03`/`CODE-REVIEW`）。 +- `tool` 事件在能从 subagent transcript 文件名识别出并行 sub-developer 轨道时 + （`sub-developer-PT-01` 这类命名）会额外带 `pt_id`（如 `"PT-01"`）。按事件自己的 + transcript_path 推断，不依赖任何 session 级共享状态——并行轨道是真并发，不能 + 用一个可变指针记"当前是哪条轨道"。 + +### 事件字段 + +| event | 实际顶层字段 | 说明 | +|---|---|---| +| `start` | `event`, `sid`, `agent`, `ts` | 会话开始 | +| `user_prompt_submit` | `event`, `sid`, `agent`, `turn_id`, `prompt_len`, `ts` | 新 turn 边界；AgentLens 侧按 `1 Trace = 1 Turn` 生成 trace | +| `tool` | `event`, `sid`, `agent`, `tool`, `ms`, `transcript_path`, `turn_id`, `ts`, `skill`, `rule`, `cwd`, `call_id`, `message_id`, `tool_details`, `task_slug`?, `stage`?, `pt_id`? | 工具调用与耗时 | +| `usage` | `event`, `sid`, `agent`, `tool`, `tokens`, `model`, `transcript_path`, `source_offset`, `turn_id`, `message_id`, `ts`, `task_slug`?, `stage`? | transcript 增量 token / 模型 / 消息归属 | +| `stop` | `event`, `sid`, `agent`, `tokens`, `model`, `transcript_path`, `source_offset`, `turn_id`, `message_id`, `ts`, `task_slug`?, `stage`? | 会话尾部 flush | +| `error` | `event`, `sid`, `phase`, `error`, `x_traceback`, `ts` | hook 自身异常记录 | +| `stage_transition` | `event`, `sid`, `task_slug`, `stage`, `status`?, `executor`?, `retry_count`?, `review_result`?, `ts` | devflow `workflow-state.json` 某个 stage 的 status/retry_count/review_result 发生变化时触发（仅 devflow 项目） | + +### 可选字段说明 + +- `tool.ms`: Pre/Post 未成功配对时可能为 `null` +- `tool.call_id`: 仅当工具调用存在 call id 时出现 +- `tool.message_id`: 仅当成功和 transcript 中的 message 关联上时出现 +- `tool.tool_details`: 仅当 transcript 中能还原出更细工具上下文时出现 +- `usage.message_id`: 仅当 usage 对应的 transcript message 可识别时出现 +- `usage.model`: transcript 中能识别模型名时出现 +- `usage.cost_usd`: 只有模型价格命中价格表时才会出现。价格表 = 内置 `config/pricing.json` 与用户覆盖文件（默认 `/.claude/agent-observability/pricing.overrides.json`，可用 `AOBS_PRICING_OVERRIDES_PATH` 指定）的字段级合并结果；覆盖只对该文件写入**之后**产生的新事件生效，不回溯重算历史成本（见 `quickstart.md` 的「自定义模型定价覆盖」） +- `stop.cost_usd`: 只有当前 stop 事件对应 usage 能估算成本时才会出现 +- `stop.cost_session_usd`: 只有 stop 汇总阶段能反算出整个 session 成本时才会出现 + +### 完整示例 + +换成更容易读的多行 JSON。下面仍然是当前 `metrics.ndjson` 里的实际数据，只是长文本字段做了截断。 + +`start` + +```json +{ + "event": "start", + "sid": "f6435d63-b4d5-44f2-b4d6-ae73d6c140fd", + "agent": "main", + "ts": 1784015331.87806 +} +``` + +`user_prompt_submit` + +```json +{ + "event": "user_prompt_submit", + "sid": "f6435d63-b4d5-44f2-b4d6-ae73d6c140fd", + "agent": "main", + "turn_id": "turn-1784015342075", + "prompt_len": 136, + "ts": 1784015342.076438 +} +``` + +`usage` + +```json +{ + "event": "usage", + "sid": "f6435d63-b4d5-44f2-b4d6-ae73d6c140fd", + "agent": "main", + "tool": "Read", + "tokens": { + "input": 31110, + "output": 594, + "cache_read": 3072, + "total": 31704 + }, + "model": "hy3-ioa", + "transcript_path": "/Users/rachel/.claude/projects/Users-rachel-skillhub-tokentrack-mr/f6435d63-b4d5-44f2-b4d6-ae73d6c140fd.jsonl", + "source_offset": 17423, + "turn_id": "turn-1784015342075", + "message_id": "d187771c6eef4520a530564dd2a73e38", + "ts": 1784015355.413464 +} +``` + +`tool` 调用参数型 + +```json +{ + "event": "tool", + "sid": "f6435d63-b4d5-44f2-b4d6-ae73d6c140fd", + "agent": "main", + "tool": "Read", + "ms": null, + "transcript_path": "/Users/rachel/.claude/projects/Users-rachel-skillhub-tokentrack-mr/f6435d63-b4d5-44f2-b4d6-ae73d6c140fd.jsonl", + "turn_id": "turn-1784015342075", + "ts": 1784015355.412617, + "skill": [], + "rule": [ + "global" + ], + "cwd": "/Users/rachel/skillhub-tokentrack-mr", + "call_id": "chatcmpl-tool-9e50f7ec1d7e4a27", + "message_id": "d187771c6eef4520a530564dd2a73e38", + "tool_details": { + "call_id": "chatcmpl-tool-9e50f7ec1d7e4a27", + "arguments": "{\"file_path\": \"/Users/rachel/skillhub-tokentrack-mr/assets/devflow.defaults.yaml\"}", + "arguments_display_text": "assets/devflow.defaults.yaml" + } +} +``` + +`tool` 返回结果型 + +```json +{ + "event": "tool", + "sid": "f6435d63-b4d5-44f2-b4d6-ae73d6c140fd", + "agent": "main", + "tool": "Bash", + "ms": 759, + "transcript_path": "/Users/rachel/.claude/projects/Users-rachel-skillhub-tokentrack-mr/f6435d63-b4d5-44f2-b4d6-ae73d6c140fd.jsonl", + "turn_id": "turn-1784015342075", + "ts": 1784015356.071573, + "skill": [], + "rule": [ + "global" + ], + "cwd": "/Users/rachel/skillhub-tokentrack-mr", + "call_id": "chatcmpl-tool-898d45e0999b3e3c", + "message_id": "93b617ceebaa49458169b89b4ecae17b", + "tool_details": { + "call_id": "chatcmpl-tool-898d45e0999b3e3c", + "result_content": "Command: cd /Users/rachel/skillhub-tokentrack-mr && ls -la .claude/teams/ ...", + "raw_response": { + "exitCode": 1, + "signal": null, + "interrupted": false, + "sandboxDenied": false, + "stderrBytesTruncated": 0, + "stdoutBytesTruncated": 0, + "tool_error_code": "8002", + "is_error": true, + "error": "Command: cd /Users/rachel/skillhub-tokentrack-mr && ls -la .claude/teams/ ..." + }, + "output_text": "Command: cd /Users/rachel/skillhub-tokentrack-mr && ls -la .claude/teams/ ...", + "original_message_id": "d187771c6eef4520a530564dd2a73e38", + "next_message_id": "93b617ceebaa49458169b89b4ecae17b", + "message_id_reassigned": true + } +} +``` + +`stop` + +```json +{ + "event": "stop", + "sid": "f6435d63-b4d5-44f2-b4d6-ae73d6c140fd", + "agent": "test-engineer", + "tokens": { + "input": 128290, + "output": 665, + "cache_read": 128192, + "total": 128955 + }, + "model": "hy3-ioa", + "transcript_path": "/Users/rachel/.claude/projects/Users-rachel-skillhub-tokentrack-mr/f6435d63-b4d5-44f2-b4d6-ae73d6c140fd.jsonl", + "source_offset": 1157291, + "turn_id": "turn-1784015342075", + "message_id": "51e4fae9553d4d0995e7a599f3b2ef2d", + "ts": 1784018405.467215 +} +``` + +`error` + +```json +{ + "event": "error", + "sid": "6e8cd633-1a2f-4488-b1df-1eef168009c3", + "phase": "post", + "error": "AttributeError: module 'core.transcript_runtime' has no attribute 'transcript_path'", + "x_traceback": "Traceback ...", + "ts": 1784183163.4314518 +} +``` + +`stage_transition`（仅 devflow 项目；见 `core/devflow.py`） + +```json +{ + "event": "stage_transition", + "sid": "f6435d63-b4d5-44f2-b4d6-ae73d6c140fd", + "task_slug": "fix-token-bypass_20260911_0900", + "stage": "CODE-REVIEW", + "status": "failed", + "executor": "code-reviewer", + "retry_count": 1, + "review_result": "failed", + "ts": 1784015412.223 +} +``` + +### tokens + +```json +{"input": 1200, "output": 180, "cache_read": 9000, "cache_creation": 0, "total": 1380} +``` + +- `total`: 统一按 `input + output` 计算 +- `cache_read` / `cache_creation`: 保留给成本估算和缓存命中分析使用 + +### tool_details + +`tool` 事件里的 `tool_details` 是一个可选嵌套对象，当前实现里可能包含： + +- `call_id` +- `arguments` +- `arguments_display_text` +- `result_content` +- `raw_response` +- `output_text` +- `original_message_id` +- `next_message_id` +- `message_id_reassigned` + +常见示例： + +```json +{ + "call_id": "chatcmpl-tool-9e50f7ec1d7e4a27", + "arguments": "{\"file_path\": \"/path/to/file\"}", + "arguments_display_text": "path/to/file" +} +``` + +### AgentLens sidecar + +`.state.json[sid]._agentlens` 只作为内部状态使用，核心字段： + +- `enabled`: AgentLens 上报是否可用 +- `last_error`: 最近一次降级原因 +- `current_turn.carrier.traceparent`: 当前 turn 的 trace context +- `current_turn.subagent_spans`: 同一 turn 下的子 agent span carrier +- `current_turn.agent_spans`: 同一 turn 下按 agent 聚合的 span carrier 与累计统计 +- `current_turn.step_spans`: `(agent, message_id)` 级 step span registry +- `turn_history`: 已结束 turn 的摘要 + +### Hook 接入 + +参考 `templates/settings-hook.json`，必须启用： + +- `SessionStart` +- `UserPromptSubmit` +- `PreToolUse` +- `PostToolUse` +- `Stop` + +### 兼容 + +- 旧日志若在 `tool` 事件里直接携带 `tokens` / `cost_usd`，按 legacy usage 处理。 +- 不理解 `turn_id` 的下游消费者可安全忽略。 diff --git a/.claude/skills/agent-observability/scripts/__init__.py b/.claude/skills/agent-observability/scripts/__init__.py new file mode 100644 index 0000000..52073be --- /dev/null +++ b/.claude/skills/agent-observability/scripts/__init__.py @@ -0,0 +1 @@ +"""agent-observability 的脚本包。""" diff --git a/.claude/skills/agent-observability/scripts/build_dashboard_data.py b/.claude/skills/agent-observability/scripts/build_dashboard_data.py new file mode 100644 index 0000000..e33f816 --- /dev/null +++ b/.claude/skills/agent-observability/scripts/build_dashboard_data.py @@ -0,0 +1,624 @@ +#!/usr/bin/env python3 +"""把 `logs/metrics.ndjson` + `artifacts/*/workflow-state.json` + `.state.json` + +`hooks/logs/auto-dispatch.log` 聚合成看板需要的一份 `dashboard-data.json` 快照。 + +这是本地看板的数据源，不是 hook 链路的一部分——hook 只管往 metrics.ndjson 追加事件， +这个脚本单独、按需运行（比如每次想看一眼看板之前手动跑一次，或者配合文件监听器）。 +复用 core.devflow 的 stage_snapshot 做 workflow-state.json 的 schema 归一化， +不在这里重新实现一遍 Classic/Portable 的字段差异。 + +用法： + python3 build_dashboard_data.py --project-root <项目根目录> --out dashboard/dashboard-data.json + python3 build_dashboard_data.py --metrics-path ~/other/metrics.ndjson --state-path ~/other/.state.json + python3 build_dashboard_data.py --top-slow 5 + +不传 --metrics-path / --state-path 时，仍从 /logs/ 下的默认文件读取； +两者均支持 ~ 展开，输出 source 会如实反映实际读取路径。 + +--top-slow N 默认不开启，开启后只在终端额外打印耗时最长的 N 次 tool 事件， +不写入 dashboard-data.json（输出结构与不开时完全一致）。 + +没有任何真实数据时（hook 刚接上、还没跑过 session），会输出一份全空但结构合法的快照， +不会报错、也不会伪造数据。 +""" +from __future__ import annotations + +import argparse +import json +import sys +from collections import defaultdict +from pathlib import Path +from typing import Any + +SCRIPTS_DIR = Path(__file__).resolve().parent +if str(SCRIPTS_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPTS_DIR)) + +from core import devflow as dv # type: ignore + +try: # emitter 只在"未定价模型"统计里用到，导入失败只让这一项退化为 [] + from core import emitter as em # type: ignore +except Exception: # pragma: no cover - 目标机缺依赖时的兜底 + em = None # type: ignore + +# devflow 两套 schema 的 stage 名 -> 展示用 (label, name)。取不到的 stage 用 key 本身兜底。 +STAGE_META: dict[str, tuple[str, str]] = { + "PHASE-0": ("P0", "初始化 + 判定大小"), + "SOLO": ("SOLO", "单 agent 全流程"), + "TASK-01": ("T01", "需求分析 + 澄清"), + "REQUIREMENT": ("REQ", "需求分析 + 澄清"), + "TASK-02": ("T02", "技术方案"), + "DESIGN": ("DES", "技术方案"), + "TASK-03": ("T03", "代码实现"), + "IMPLEMENT": ("IMPL", "代码实现"), + "CODE-REVIEW": ("CR", "代码审查"), + "REVIEW": ("REV", "代码审查"), + "TASK-04": ("T04", "E2E 测试"), + "TEST": ("TEST", "E2E 测试"), + "TASK-05": ("T05", "知识沉淀"), + "KNOWLEDGE": ("KNOW", "知识沉淀"), + "SUMMARY": ("汇总", "最终汇总"), +} +# 两套 schema 各自的阶段顺序，用来按正确顺序渲染 stepper（不能直接遍历 dict，顺序不保证）。 +CLASSIC_ORDER = ["PHASE-0", "TASK-01", "TASK-02", "TASK-03", "CODE-REVIEW", "TASK-04", "TASK-05"] +# SOLO 完成后，small 任务会由 solo-developer 合并执行 TASK-05 知识沉淀（真实运行验证过， +# 不是理论上可选的分支）；stepper 顺序必须把它列进去，否则会把已完成的阶段悄悄漏掉。 +CLASSIC_SOLO_ORDER = ["PHASE-0", "SOLO", "TASK-05"] +PORTABLE_ORDER = ["PHASE-0", "REQUIREMENT", "DESIGN", "IMPLEMENT", "REVIEW", "TEST", "KNOWLEDGE", "SUMMARY"] +PORTABLE_SOLO_ORDER = ["PHASE-0", "SOLO", "SUMMARY"] + + +def read_ndjson(path: Path) -> list[dict[str, Any]]: + records: list[dict[str, Any]] = [] + if not path.is_file(): + return records + with path.open("r", encoding="utf-8") as fp: + for line in fp: + line = line.strip() + if not line.startswith("{"): + continue + try: + obj = json.loads(line) + except Exception: + continue + if isinstance(obj, dict): + records.append(obj) + return records + + +def safe_load_json(path: Path) -> Any: + try: + return json.loads(path.read_text("utf-8")) + except Exception: + return None + + +def day_of(ts: float) -> str: + from datetime import datetime, timezone + return datetime.fromtimestamp(ts, tz=timezone.utc).strftime("%Y-%m-%d") + + +def tool_call_failed(raw_response: Any) -> bool: + """判断一次工具调用是否失败。 + + 真实 CodeBuddy CLI 的 transcript（`function_call_result.providerData.toolResult. + rawResponse`）里从来没有 `is_error` 这个布尔字段——实际信号是 `exitCode`（非 0 + 即失败）和/或 `tool_error_code`（非 "0"/空即失败）。之前只认 `is_error`，导致 + 失败统计在这个宿主上永远是 0，不管命令是否真的失败（复合命令比如 + `cmd; echo ...` 会把失败进一步掩盖成 exitCode=0，那种情况下这里也如实判定为 + 未失败——判断整条工具调用本身有没有失败，不追究命令内部的子步骤）。 + 仍然保留 `is_error` 判断，兼容其它可能真的写这个字段的宿主。 + """ + if not isinstance(raw_response, dict): + return False + if raw_response.get("is_error"): + return True + exit_code = raw_response.get("exitCode") + if isinstance(exit_code, (int, float)) and exit_code != 0: + return True + tool_error_code = raw_response.get("tool_error_code") + if isinstance(tool_error_code, str) and tool_error_code.strip() not in ("", "0"): + return True + return False + + +def build_daily_and_sessions(events: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + by_day: dict[str, dict[str, Any]] = {} + by_sid: dict[str, dict[str, Any]] = {} + by_sid_turn: dict[tuple[str, str], dict[str, Any]] = {} + # AgentLens（写 state.current_turn 的那套 tracing）默认关闭时，tool/usage 事件 + # 自身的 turn_id 永远是 None——但 user_prompt_submit 事件不依赖 AgentLens， + # 始终携带真实 turn_id（见 emitter.build_prompt_submit_event）。按 ts 排序后， + # 把它当作 turn 边界，让后续没有自带 turn_id 的 tool/usage 事件归入"当前 sid + # 最近一次打开的 turn"，而不是全部塌缩进同一个 "?" 占位桶。 + current_turn_by_sid: dict[str, str] = {} + + def day_bucket(iso: str) -> dict[str, Any]: + return by_day.setdefault(iso, { + "iso": iso, "sessionCount": 0, "input": 0, "output": 0, "cache": 0, + "cost": 0.0, "toolCalls": 0, "failures": 0, "skillHits": defaultdict(int), + "_sids": set(), + }) + + def sess_bucket(sid: str) -> dict[str, Any]: + return by_sid.setdefault(sid, { + "id": sid, "date": None, "agents": set(), "turns": set(), "toolCalls": 0, + "tokens": 0, "cost": 0.0, "first_ts": None, "last_ts": None, "models": defaultdict(int), + "status": "ok", "timeline": {}, + }) + + for rec in sorted(events, key=lambda r: r.get("ts") if isinstance(r.get("ts"), (int, float)) else 0): + event = rec.get("event") + sid = str(rec.get("sid") or "") + ts = rec.get("ts") + if not sid or not isinstance(ts, (int, float)): + continue + iso = day_of(ts) + sess = sess_bucket(sid) + sess["_seen_day"] = iso + # 不管事件类型，只要这天有这个 sid 的任何事件就算一次"当日活跃会话"。 + # 之前只在 tool/usage 分支里 add，纯对话（只有 user_prompt_submit，没有 + # 触发任何工具调用也没有 usage）的会话永远不会被计入任何一天的 + # sessionCount——总览页的"会话数"会比"会话浏览"tab 里实际展开的会话数少 + # （真实数据复现过：5 个 session 只统计出 4 个 sessionCount）。 + day_bucket(iso)["_sids"].add(sid) + if sess["first_ts"] is None or ts < sess["first_ts"]: + sess["first_ts"] = ts + sess["date"] = iso + if sess["last_ts"] is None or ts > sess["last_ts"]: + sess["last_ts"] = ts + agent = rec.get("agent") + if agent: + sess["agents"].add(str(agent)) + raw_turn_id = rec.get("turn_id") + if event == "user_prompt_submit" and raw_turn_id: + current_turn_by_sid[sid] = str(raw_turn_id) + # 提前建桶——纯对话轮次（没有触发任何工具调用，只有 usage 都没有） + # 之前只在 tool/usage 分支里 setdefault，这种轮次永远不会出现在 + # timeline 里，导致 turns 计数和时间线展开的分组数对不上。 + by_sid_turn.setdefault( + (sid, str(raw_turn_id)), + {"turn": raw_turn_id, "agent": agent or "main", "events": []}, + ) + turn_id = raw_turn_id or current_turn_by_sid.get(sid) + if turn_id: + sess["turns"].add(str(turn_id)) + + if event == "tool": + day = day_bucket(iso) + day["toolCalls"] += 1 + sess["toolCalls"] += 1 + for s in rec.get("skill") or []: + day["skillHits"][str(s)] += 1 + tool_details = rec.get("tool_details") or {} + raw_response = tool_details.get("raw_response") if isinstance(tool_details, dict) else None + is_err = tool_call_failed(raw_response) + if is_err: + day["failures"] += 1 + sess["status"] = "error" + key = (sid, str(turn_id or "?")) + turn = by_sid_turn.setdefault(key, {"turn": turn_id or "?", "agent": agent or "main", "events": []}) + turn["events"].append({ + "kind": "tool", "tool": rec.get("tool"), + "ms": rec.get("ms") if isinstance(rec.get("ms"), (int, float)) else 0, + "err": is_err, + "agent": str(agent) if agent else "main", + }) + elif event == "usage": + day = day_bucket(iso) + tokens = rec.get("tokens") or {} + day["input"] += int(tokens.get("input") or 0) + day["output"] += int(tokens.get("output") or 0) + day["cache"] += int(tokens.get("cache_read") or 0) + cost = rec.get("cost_usd") + if isinstance(cost, (int, float)): + day["cost"] += float(cost) + sess["cost"] += float(cost) + total_tok = int(tokens.get("input") or 0) + int(tokens.get("output") or 0) + sess["tokens"] += total_tok + model = rec.get("model") + if model: + sess["models"][str(model)] += 1 + key = (sid, str(turn_id or "?")) + turn = by_sid_turn.setdefault(key, {"turn": turn_id or "?", "agent": agent or "main", "events": []}) + turn["events"].append({ + "kind": "usage", "tokens": total_tok, + "agent": str(agent) if agent else "main", + }) + elif event == "error": + sess["status"] = "error" + + for (sid, _turn_key), turn in by_sid_turn.items(): + sess = by_sid.setdefault(sid, sess_bucket(sid)) + sess["timeline"].setdefault(_turn_key, turn) + + daily = [] + for iso in sorted(by_day.keys()): + d = by_day[iso] + daily.append({ + "iso": iso, "sessionCount": len(d["_sids"]), "input": d["input"], "output": d["output"], + "cache": d["cache"], "cost": round(d["cost"], 4), "toolCalls": d["toolCalls"], + "failures": d["failures"], "skillHits": dict(d["skillHits"]), + }) + + sessions = [] + for sid, s in by_sid.items(): + if s["first_ts"] is None: + continue + duration = int((s["last_ts"] or s["first_ts"]) - s["first_ts"]) + model = max(s["models"].items(), key=lambda kv: kv[1])[0] if s["models"] else None + timeline_sorted = sorted(s["timeline"].values(), key=lambda t: str(t.get("turn") or "")) + for idx, t in enumerate(timeline_sorted, start=1): + t["turn"] = idx + sessions.append({ + "id": sid, "date": s["date"], "agent": ", ".join(sorted(s["agents"])) or "main", + "turns": len(s["turns"]) or len(timeline_sorted), "toolCalls": s["toolCalls"], + "tokens": s["tokens"], "cost": round(s["cost"], 4), "duration": duration, + "model": model, "status": s["status"], "timeline": timeline_sorted, + }) + sessions.sort(key=lambda s: s["date"] or "", reverse=True) + return daily, sessions + + +def build_model_costs(events: list[dict[str, Any]]) -> list[dict[str, Any]]: + totals: dict[str, float] = defaultdict(float) + for rec in events: + if rec.get("event") != "usage": + continue + model = rec.get("model") + cost = rec.get("cost_usd") + if model and isinstance(cost, (int, float)): + totals[str(model)] += float(cost) + rows = [{"name": name, "cost": round(cost, 4)} for name, cost in totals.items()] + rows.sort(key=lambda r: r["cost"], reverse=True) + return rows + + +def build_unpriced_models(events: list[dict[str, Any]]) -> list[dict[str, Any]]: + """聚合"完全没命中价格表"的模型（D4：`emitter.is_unpriced(model)` 为 True）。 + + 只**统计**不重算：`cost_usd` 在 hook 期就写进事件了，这里不会（也不该）回头 + 按新价格补算历史事件（D3：覆盖只对新事件生效）。判定入口复用 emitter， + 不在本文件里重写一套模型名匹配逻辑，避免与 hook 期的口径分叉。 + + 模型名缺失/为空的 usage 事件无法归因到某个模型，直接跳过不计数。 + """ + if em is None: + return [] + counts: dict[str, int] = defaultdict(int) + for rec in events: + if rec.get("event") != "usage": + continue + model = rec.get("model") + if not model or not str(model).strip(): + continue + name = str(model) + try: + if em.is_unpriced(name): + counts[name] += 1 + except Exception: + continue + rows = [{"name": name, "usageEvents": count} for name, count in counts.items()] + # 按 (-事件数, 名字) 排序：避免看板顺序抖动，两次生成的快照可直接逐字节比对。 + rows.sort(key=lambda r: (-r["usageEvents"], r["name"])) + return rows + + +def build_failures(events: list[dict[str, Any]]) -> list[dict[str, Any]]: + agg: dict[tuple[str, str], dict[str, Any]] = {} + for rec in events: + if rec.get("event") != "tool": + continue + tool_details = rec.get("tool_details") or {} + raw_response = tool_details.get("raw_response") if isinstance(tool_details, dict) else None + if not tool_call_failed(raw_response): + continue + tool = str(rec.get("tool") or "unknown") + code = str(raw_response.get("tool_error_code") or raw_response.get("exitCode") or "error") + key = (tool, code) + row = agg.setdefault(key, {"tool": tool, "code": code, "count": 0, "last": None, "_last_ts": 0.0}) + row["count"] += 1 + ts = rec.get("ts") + if isinstance(ts, (int, float)) and ts > row["_last_ts"]: + row["_last_ts"] = ts + row["last"] = day_of(ts) + rows = list(agg.values()) + for row in rows: + row.pop("_last_ts", None) + rows.sort(key=lambda r: r["count"], reverse=True) + return rows + + +def build_top_slow(events: list[dict[str, Any]], limit: int | None = None) -> list[dict[str, Any]]: + """取耗时最长的 N 次 tool 事件，按 ms 降序。 + + 只做"读"：不改动 events，也不参与 dashboard-data.json 的任何字段——调用方拿 + 结果去打印即可。`ms` 的兜底口径与 build_daily_and_sessions 保持一致（缺失或 + 非数字按 0 计），否则同一份日志会得出两个互相矛盾的工具耗时视图。 + """ + if not isinstance(limit, int) or limit <= 0: + return [] + rows: list[dict[str, Any]] = [] + for rec in events: + if rec.get("event") != "tool": + continue + rows.append({ + "tool": str(rec.get("tool") or "unknown"), + "ms": rec.get("ms") if isinstance(rec.get("ms"), (int, float)) else 0, + }) + rows.sort(key=lambda r: r["ms"], reverse=True) + return rows[:limit] + + +def format_top_slow(rows: list[dict[str, Any]]) -> str: + """把 top-slow 结果渲染成终端文本；没有可展示的行时返回空串（由调用方决定 + 是否打印，避免没有任何 tool 事件时空打一个表头）。""" + if not rows: + return "" + width = max(len(str(r["tool"])) for r in rows) + lines = [f"top-slow {len(rows)} tool calls (by ms):"] + for idx, row in enumerate(rows, start=1): + lines.append(f" {idx}. {str(row['tool']):<{width}} {row['ms']}ms") + return "\n".join(lines) + + +def build_skills(state: dict[str, Any]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + totals: dict[str, int] = defaultdict(int) + never_used: dict[str, dict[str, Any]] = {} + for sid, sess in state.items(): + if sid.startswith("_") or not isinstance(sess, dict): + continue + for name, rec in (sess.get("skills") or {}).items(): + if not isinstance(rec, dict): + continue + totals[name] += int(rec.get("count") or 0) + for name, rec in (sess.get("rules") or {}).items(): + if not isinstance(rec, dict) or int(rec.get("count") or 0) > 0: + continue + never_used[name] = {"name": name, "reason": "no_paths", "detail": "从未被路径推断或 active-rule 命中"} + for name, count in totals.items(): + if count == 0: + never_used.setdefault(name, {"name": name, "reason": "no_paths", "detail": "静态扫描到，但从未被任何工具调用命中"}) + skill_rows = [{"name": name, "count": count} for name, count in totals.items() if count > 0] + skill_rows.sort(key=lambda r: r["count"], reverse=True) + return skill_rows, sorted(never_used.values(), key=lambda r: r["name"]) + + + +# "team-lead" 是 CodeBuddy 原生 team 基础设施里 lead/orchestrator session 自己的 +# mailbox 名字，语义上就是 main（core/agent_identity.py::normalize_role_name 把它 +# 和 "main" 归为同一类）；不是一个真实存在的子 agent，不应出现在"派发目标"里。 +_DISPATCH_EXCLUDE_AGENTS = {"main", "team-lead"} + + +def build_dispatch(state: dict[str, Any]) -> list[dict[str, Any]]: + totals: dict[str, int] = defaultdict(int) + for sid, sess in state.items(): + if sid.startswith("_") or not isinstance(sess, dict): + continue + for entry in sess.get("agent_history") or []: + if not isinstance(entry, dict): + continue + evidence = str(entry.get("evidence") or "") + agent = entry.get("agent") + if not agent or agent in _DISPATCH_EXCLUDE_AGENTS: + continue + # 只统计真正代表"新派发"的证据：工具调用直接触发的 dispatch@， + # 或 inbox 扫描独立确认的 "main 派给了谁"（inbox@dispatch:*）。 + # 排除 inbox@report:*／inbox@handoff:*——那是子 agent 上报/移交， + # 不是 main 发起的新派发，计进来会把"派发次数"虚高（同一次真实 + # 派发经常先触发 dispatch@，随后又在 inbox 里被自己的上报回声一次）。 + is_dispatch = evidence.startswith("dispatch@") or evidence.startswith("inbox@dispatch:") + if is_dispatch: + totals[str(agent)] += 1 + rows = [{"agent": agent, "count": count} for agent, count in totals.items()] + rows.sort(key=lambda r: r["count"], reverse=True) + return rows + + +def build_auto_dispatch_stats(project_root: Path) -> dict[str, int]: + log_path = project_root / ".codebuddy" / "hooks" / "logs" / "auto-dispatch.log" + stats = {"auto_dispatch": 0, "fallback_to_main": 0, "passthrough": 0} + if not log_path.is_file(): + return stats + with log_path.open("r", encoding="utf-8") as fp: + for line in fp: + line = line.strip() + if not line.startswith("{"): + continue + try: + rec = json.loads(line) + except Exception: + continue + decision = str(rec.get("decision") or "") + if decision == "auto_dispatch": + stats["auto_dispatch"] += 1 + elif decision == "fallback_to_main": + stats["fallback_to_main"] += 1 + elif decision in {"passthrough"}: + stats["passthrough"] += 1 + return stats + + +def build_cost_by_task_slug(events: list[dict[str, Any]]) -> dict[str, float]: + """按 task_slug 汇总 usage/stop 事件的 cost_usd——workflow-state.json 自己不知道成本， + 这是唯一能把 devflow 运行和真实花费对上的地方（需要 emitter 已经把 task_slug 挂到事件上， + 见 core/runtime.py 的 devflow 接入）。""" + totals: dict[str, float] = defaultdict(float) + for rec in events: + if rec.get("event") not in {"usage", "stop"}: + continue + task_slug = rec.get("task_slug") + cost = rec.get("cost_usd") + if task_slug and isinstance(cost, (int, float)): + totals[str(task_slug)] += float(cost) + return dict(totals) + + +def build_devflow_runs(project_root: Path, cost_by_slug: dict[str, float] | None = None) -> list[dict[str, Any]]: + artifacts_root = project_root / "artifacts" + runs: list[dict[str, Any]] = [] + cost_by_slug = cost_by_slug or {} + if not artifacts_root.is_dir(): + return runs + for child in sorted(artifacts_root.iterdir()): + state_path = child / "workflow-state.json" + raw = dv.read_workflow_state(str(state_path)) + if not isinstance(raw, dict) or not isinstance(raw.get("stages"), dict): + continue + snap = dv.stage_snapshot(raw) + size_class = str(snap.get("size_class") or "medium") + # medium/large 的 workflow-state.json 也会带着 SOLO key（模板固定写入， + # 状态停在 "pending" 或 "skipped"，从未被真正执行）——不能只看 key 存不存在， + # 必须看 SOLO 阶段是不是真的跑过，否则每个 medium/large 任务都会被误判成 + # solo 路径，阶段视图漏掉 TASK-01/02/03/CODE-REVIEW/TASK-04。 + solo_info = snap["stages"].get("SOLO") + solo_status = str(solo_info.get("status") or "") if isinstance(solo_info, dict) else "" + solo_actually_ran = solo_status not in {"", "pending", "skipped"} + is_solo = size_class == "small" or (size_class not in {"small", "medium", "large"} and solo_actually_ran) + if snap.get("schema_version") == "2.0": + order = PORTABLE_SOLO_ORDER if is_solo else PORTABLE_ORDER + else: + order = CLASSIC_SOLO_ORDER if is_solo else CLASSIC_ORDER + stages_out = [] + total_cost = cost_by_slug.get(child.name, 0.0) + run_start = None + run_end = None + overall = "completed" + for key in order: + info = snap["stages"].get(key) + if info is None: + # PHASE-0 在两套 schema 里都不会出现在 stages{} 里（它是隐式完成的： + # workflow-state.json 一旦存在，就说明 Phase 0 已经跑完了）， + # 不能用"没有条目"直接兜底成 pending，那样会把已完成的阶段显示错。 + info = {"status": "completed", "retry_count": 0, "executor": None} if key == "PHASE-0" \ + else {"status": "pending", "retry_count": 0, "executor": None} + label, name = STAGE_META.get(key, (key[:4], key)) + status = str(info.get("status") or "pending") + retry_count = int(info.get("retry_count") or 0) + raw_stage = raw.get("stages", {}).get(key) if isinstance(raw.get("stages"), dict) else {} + duration = 0 + if isinstance(raw_stage, dict): + started = raw_stage.get("started_at") + completed = raw_stage.get("completed_at") + if isinstance(started, str) and isinstance(completed, str): + try: + from datetime import datetime + t0 = datetime.fromisoformat(started.replace("Z", "+00:00")) + t1 = datetime.fromisoformat(completed.replace("Z", "+00:00")) + duration = max(0, int((t1 - t0).total_seconds())) + if run_start is None or t0 < run_start: + run_start = t0 + if run_end is None or t1 > run_end: + run_end = t1 + except Exception: + duration = 0 + if status == "failed": + overall = "paused" + elif status == "in_progress" and overall != "paused": + overall = "running" + stages_out.append({ + "key": key, "label": label, "name": name, + "exec": info.get("executor") or "-", "status": status, + "retry_count": retry_count, "duration": duration, + }) + # 按真实起止时间跨度算总时长（而不是逐阶段 duration 相加）——阶段之间可能 + # 有重叠（比如 TASK-05 的 started_at 早于 SOLO 的 completed_at），相加会 + # 把重叠部分重复计入，虚高于源数据本身反映的运行时长。 + total_duration = int((run_end - run_start).total_seconds()) if run_start and run_end else 0 + runs.append({ + "slug": child.name, "size": size_class, "stages": stages_out, + "cost": round(total_cost, 4), "duration": total_duration, "overall": overall, + }) + runs.sort(key=lambda r: r["slug"], reverse=True) + return runs + + +def resolve_input_paths( + skill_root: Path, + metrics_path: str | None = None, + state_path: str | None = None, +) -> tuple[Path, Path]: + """解析 metrics.ndjson 与 .state.json 的实际读取路径。 + + 任一参数为 None 时回退到 ``/logs/`` 下的默认路径，保证未传参时 + 行为与旧版本完全一致（向后兼容，不会破坏现有看板数据源）。 + + 非 None 时按用户给定路径（支持 ``~`` 展开）取绝对路径，便于看板从非默认 + 位置（如其它会话/项目的日志目录）聚合数据。 + """ + default_metrics = skill_root / "logs" / "metrics.ndjson" + default_state = skill_root / "logs" / ".state.json" + if metrics_path: + metrics = Path(metrics_path).expanduser().absolute() + else: + metrics = default_metrics + if state_path: + state = Path(state_path).expanduser().absolute() + else: + state = default_state + return metrics, state + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--project-root", default=".", help="devflow 项目根目录（含 .codebuddy/ 和 artifacts/）") + parser.add_argument("--out", default=None, help="输出路径，默认 /scripts/dashboard/dashboard-data.json") + parser.add_argument("--metrics-path", default=None, help="覆盖 metrics.ndjson 的读取路径（默认 /logs/metrics.ndjson，支持 ~ 展开）") + parser.add_argument("--state-path", default=None, help="覆盖 .state.json 的读取路径（默认 /logs/.state.json，支持 ~ 展开）") + parser.add_argument("--top-slow", type=int, default=None, help="额外在终端打印耗时最长的 N 次 tool 事件（工具名 + 耗时 ms），默认不打印；只打印不写入 dashboard-data.json") + args = parser.parse_args() + + project_root = Path(args.project_root).expanduser().resolve() + skill_root = SCRIPTS_DIR.parent + log_path, state_path = resolve_input_paths(skill_root, args.metrics_path, args.state_path) + + # 价格表带 lru_cache：本脚本可能长期驻留（配合文件监听器），单测也会同进程多次 + # 调 main()，每次都先清一次缓存，保证覆盖文件的改动当场生效。 + if em is not None: + try: + em.clear_price_cache() + except Exception: + pass + + events = read_ndjson(log_path) + state = safe_load_json(state_path) or {} + if not isinstance(state, dict): + state = {} + + daily, sessions = build_daily_and_sessions(events) + model_costs = build_model_costs(events) + unpriced = build_unpriced_models(events) + failures = build_failures(events) + skills, never_used = build_skills(state) + dispatch = build_dispatch(state) + auto_dispatch_stats = build_auto_dispatch_stats(project_root) + cost_by_slug = build_cost_by_task_slug(events) + devflow_runs = build_devflow_runs(project_root, cost_by_slug) + + from datetime import datetime, timezone + out_data = { + "generated_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "source": { + "metrics_ndjson": str(log_path), "state_json": str(state_path), + "project_root": str(project_root), "event_count": len(events), + }, + "daily": daily, "sessions": sessions, "modelCosts": model_costs, "unpricedModels": unpriced, + "failures": failures, + "skills": skills, "neverUsed": never_used, "dispatch": dispatch, + "autoDispatchStats": auto_dispatch_stats, "devflowRuns": devflow_runs, + } + + out_path = Path(args.out).expanduser().resolve() if args.out else (SCRIPTS_DIR / "dashboard" / "dashboard-data.json") + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(json.dumps(out_data, ensure_ascii=False, indent=2), encoding="utf-8") + print(f"wrote {out_path} ({len(events)} events, {len(sessions)} sessions, {len(devflow_runs)} devflow runs)") + # top-slow 放在原输出之后：先保证不传参时的终端输出与 JSON 行为完全不变。 + top_slow_block = format_top_slow(build_top_slow(events, args.top_slow)) + if top_slow_block: + print(top_slow_block) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.claude/skills/agent-observability/scripts/core/__init__.py b/.claude/skills/agent-observability/scripts/core/__init__.py new file mode 100644 index 0000000..1d6ad5e --- /dev/null +++ b/.claude/skills/agent-observability/scripts/core/__init__.py @@ -0,0 +1,8 @@ +"""agent-observability 的核心运行时模块集合。 + +这里放的是 hook 主流程真正依赖的内部实现： +- 采集与 transcript 解析 +- 状态读写与去重 +- skill/rule 扫描 +- AgentLens trace 镜像 +""" diff --git a/.claude/skills/agent-observability/scripts/core/agent_identity.py b/.claude/skills/agent-observability/scripts/core/agent_identity.py new file mode 100644 index 0000000..270ff90 --- /dev/null +++ b/.claude/skills/agent-observability/scripts/core/agent_identity.py @@ -0,0 +1,587 @@ +"""Agent 身份解析层。 + +这一层负责统一解析“当前是谁在工作、又把任务派给了谁”： +- 从 tool 调用参数里提取 agent 线索 +- 从 team inbox / mailbox 中补偿推断 agent 身份 +- 把解析结果写回 session state，供后续日志归因复用 +""" +from __future__ import annotations + +import json +import os +import re +import time +from datetime import datetime, timezone +from functools import lru_cache +from pathlib import Path +from typing import Any, Optional + +from . import scanner, state as st + +AGENT_PATH_RE = re.compile(r"\.codebuddy/agents/([a-zA-Z0-9_\-]+)(?:\.md)?", re.I) +DISPATCH_TOOLS = {"Task", "task", "Agent", "DeferExecuteTool"} +MESSAGE_TOOLS = {"send_message", "SendMessage"} +TEAM_PREFIX = "multi-agents-devflow-" +ROLE_INSTANCE_RE = re.compile(r"^(?P[a-z0-9][a-z0-9\-]*?)-\d+$", re.I) +INITIAL_ASSIGNMENT_RE = re.compile( + r"Initial task assignment for (?P[a-z0-9][a-z0-9\-]*)", + re.I, +) +ROLE_DECLARATION_RE = re.compile( + r"角色[：:]\s*(?P[a-z0-9][a-z0-9\-]*)", + re.I, +) +# devflow TASK-03 并行 fan-out 时，developer 会以 Task(name="sub-developer-PT-01", ...) +# 派发多条并行轨道。轨道号只在 transcript 文件名（= Task 的 name 参数）里出现， +# 按 subagent_name 归一化后会被折成同一个 "developer" —— 这是有意的（用于按角色汇总）， +# 轨道号需要单独提取，见 pt_id_from_transcript_path。 +PT_TRACK_RE = re.compile(r"-((?:PT|pt)-\d+)$") + + +def normalize_role_name(name: str | None) -> Optional[str]: + """把角色名归一化成稳定标识，例如把实例名折叠回基础角色名。""" + if not isinstance(name, str): + return None + cand = name.strip().lower() + if not cand: + return None + if cand.endswith(".json"): + cand = cand[:-5] + if "@" in cand: + cand = cand.split("@", 1)[0] + if cand in {"team-lead", "main"}: + return "main" + match = ROLE_INSTANCE_RE.match(cand) + if match: + cand = match.group("base") + return cand or None + + +def resolve_teams_root() -> Path: + """解析 CodeBuddy team inbox 的根目录。""" + config_dir = (os.environ.get("CODEBUDDY_CONFIG_DIR") or "").strip() + if config_dir: + return Path(config_dir).expanduser() / "teams" + return Path.home() / ".codebuddy" / "teams" + + +def resolve_team_dir(cwd: str, sid: str, cached_team_dir: str | None = None) -> Path | None: + """根据 session id 和 cwd 找到当前会话对应的 team 目录。""" + if cached_team_dir: + cached = Path(cached_team_dir).expanduser() + if (cached / "config.json").is_file(): + cfg = _load_json(cached / "config.json") + if isinstance(cfg, dict) and sid and str(cfg.get("leadSessionId") or "") == sid: + return cached + + if not sid: + return None + + teams_root = resolve_teams_root() + if not teams_root.is_dir(): + return None + + best_dir: Path | None = None + best_created = -1 + for team_dir in teams_root.iterdir(): + if not team_dir.is_dir() or not team_dir.name.startswith(TEAM_PREFIX): + continue + config = _load_json(team_dir / "config.json") + if not isinstance(config, dict): + continue + if str(config.get("leadSessionId") or "") != sid: + continue + if not _config_matches_cwd(config, cwd): + continue + + created_at = _safe_int(config.get("createdAt")) + if created_at <= 0: + try: + created_at = int(team_dir.stat().st_mtime * 1000) + except Exception: + created_at = 0 + if created_at > best_created: + best_dir = team_dir + best_created = created_at + return best_dir + + +def read_recent_messages( + team_dir: Path, + offsets: dict[str, int] | None = None, + *, + max_per_mailbox: int = 30, +) -> tuple[list[dict[str, Any]], dict[str, int]]: + """按 mailbox 增量读取最近消息，并返回新的 offset 游标。""" + inbox_dir = team_dir / "inboxes" + if not inbox_dir.is_dir(): + return [], offsets or {} + + prev_offsets = offsets or {} + new_offsets: dict[str, int] = {} + messages: list[dict[str, Any]] = [] + + for inbox_file in sorted(inbox_dir.glob("*.json")): + raw = _load_json(inbox_file) + if not isinstance(raw, list): + raw = [] + + total = len(raw) + prev = _safe_int(prev_offsets.get(inbox_file.name)) + if prev < 0 or prev > total: + prev = 0 + + start = prev + if start == 0 and total > max_per_mailbox: + start = total - max_per_mailbox + + mailbox_name = inbox_file.stem + mailbox_role = normalize_role_name(mailbox_name) + for idx, item in enumerate(raw[start:], start=start): + norm = _normalize_message(item, mailbox_name, mailbox_role, idx) + if norm: + messages.append(norm) + + new_offsets[inbox_file.name] = total + + messages.sort(key=lambda x: (x.get("ts") or 0.0, x.get("mailbox_name") or "", x.get("index") or 0)) + return messages, new_offsets + + +def infer_identity_from_messages(messages: list[dict[str, Any]]) -> tuple[Optional[str], Optional[str], dict[str, Any]]: + """根据 inbox 消息流推断当前 agent 与派发目标。""" + current: Optional[str] = None + dispatched: Optional[str] = None + evidence: Optional[str] = None + + for msg in messages: + mailbox_name = str(msg.get("mailbox_name") or "") + mailbox_role = normalize_role_name(msg.get("mailbox_role")) + from_role = normalize_role_name(msg.get("from_role")) + payload = msg.get("payload") if isinstance(msg.get("payload"), dict) else {} + next_target = _extract_next_target(payload) + + if from_role == "main" and mailbox_role and mailbox_role != "main": + if not msg.get("is_shutdown"): + current = mailbox_role + dispatched = mailbox_role + evidence = f"dispatch:{mailbox_name}" + continue + + if mailbox_name == "team-lead" and from_role and from_role not in {"main", "system"}: + report_role = normalize_role_name( + payload.get("from_role") if isinstance(payload, dict) else None + ) or from_role + if next_target: + current = next_target + dispatched = next_target + evidence = f"handoff:{report_role}->{next_target}" + else: + current = report_role + evidence = f"report:{report_role}" + + meta: dict[str, Any] = {"messages_seen": len(messages)} + if evidence: + meta["evidence"] = evidence + return current, dispatched, meta + + +def safe_agent_from_provider(record: dict[str, Any]) -> str | None: + """当 transcript 自己带有 providerData 时，直接读取其中的 agent。""" + provider = record.get("providerData") + if not isinstance(provider, dict): + return None + agent = provider.get("agent") + if isinstance(agent, str) and agent.strip(): + return normalize_role_name(agent.strip()) + return None + + +def role_from_content_items(content: Any) -> str | None: + """从 transcript content 文本中提取更具体的业务角色名。""" + if not isinstance(content, list): + return None + for item in content: + if not isinstance(item, dict): + continue + text = item.get("text") + if not isinstance(text, str): + continue + for regex in (INITIAL_ASSIGNMENT_RE, ROLE_DECLARATION_RE): + match = regex.search(text) + if not match: + continue + role = normalize_role_name(match.group("role")) + if role: + return role + return None + + +@lru_cache(maxsize=256) +def role_for_subagent_transcript(transcript_path: str) -> str | None: + """从 subagent transcript 前几行推断更准确的角色名。""" + path = Path(transcript_path) + try: + with path.open("r", encoding="utf-8") as fp: + for _ in range(6): + line = fp.readline() + if not line: + break + try: + record = json.loads(line) + except Exception: + continue + + # 对 subagent transcript，优先信任务分配文本里的具体角色， + # 再回退到 providerData.agent，避免 general-purpose 覆盖业务角色。 + content_role = role_from_content_items(record.get("content")) + if content_role: + return content_role + + provider_agent = safe_agent_from_provider(record) + if provider_agent: + return provider_agent + except Exception: + return None + return None + + +def agent_for_transcript_path(transcript_path: str, fallback: str | None = None) -> str: + """把 transcript 路径映射回实际对应的 agent 身份。""" + parts = Path(transcript_path).parts + if "subagents" in parts: + return role_for_subagent_transcript(transcript_path) or Path(transcript_path).stem + return fallback or "main" + + +def pt_id_from_transcript_path(transcript_path: str) -> str | None: + """从并行 sub-developer 的 transcript 文件名里提取 PT 轨道号（如 "PT-01"）。 + + 按事件自己的 transcript_path 推断，而不是写一个共享的 session 级"当前 PT"指针—— + devflow 一次可以并行派发最多 6 条 sub-developer 轨道，它们在同一个 sid 下 + 真·并发运行，任何"当前是哪条轨道"的可变共享状态在并发场景下都是错的。 + """ + parts = Path(transcript_path).parts + if "subagents" not in parts: + return None + match = PT_TRACK_RE.search(Path(transcript_path).stem) + return match.group(1).upper() if match else None + + +def merge_agent_identity_from_inbox( + state_path: Path, + sid: str, + cwd: str, + active_agent: str | None, + dispatched: str | None, +) -> tuple[str, str | None]: + """在基于 tool 的推断之上，再叠加 mailbox/inbox 的证据。""" + + def _update(state: dict[str, Any]) -> tuple[str, str | None]: + sess = st.ensure_session(state, sid) + cached_team_dir = str(sess.get("_team_dir") or "").strip() or None + offsets = sess.get("_inbox_offsets") + if not isinstance(offsets, dict): + offsets = {} + + effective_agent = active_agent or str(sess.get("current_agent") or "main") + effective_dispatched = dispatched + team_dir = resolve_team_dir(cwd, sid, cached_team_dir) + if team_dir is None: + return effective_agent, effective_dispatched + + sess["_team_dir"] = str(team_dir) + messages, new_offsets = read_recent_messages(team_dir, offsets) + sess["_inbox_offsets"] = new_offsets + inferred_current, inferred_dispatched, meta = infer_identity_from_messages(messages) + if inferred_current: + effective_agent = inferred_current + if inferred_dispatched: + effective_dispatched = inferred_dispatched + if inferred_current or inferred_dispatched: + target_agent = inferred_current or inferred_dispatched + if target_agent: + sess["current_agent"] = target_agent + hist = sess.setdefault("agent_history", []) + hist.append({ + "ts": time.time(), + "agent": target_agent, + "evidence": f"inbox@{meta.get('evidence') or 'inbox'}", + }) + return effective_agent, effective_dispatched + + return st.update_state_locked(state_path, _update) + + +def resolve_active_agent_for_event( + *, + state_path: Path, + sid: str, + cwd: str, + data: dict[str, Any], +) -> tuple[str, str | None]: + """优先用 tool 线索、其次用 inbox 线索，解析事件对应的 agent。""" + tracker = AgentIdentityResolver() + active_agent, dispatched = tracker.track(state_path, sid, data) + if scanner.is_brainstorming_call(data): + + def _update(state_data: dict[str, Any]) -> None: + sess = st.ensure_session(state_data, sid) + sess["current_agent"] = "main" + + st.update_state_locked(state_path, _update) + return "main", dispatched + return merge_agent_identity_from_inbox( + state_path, + sid, + cwd, + active_agent, + dispatched, + ) + + +class AgentIdentityResolver: + """基于 tool 调用内容做一轮 agent 身份推断。""" + def __init__(self, known_agents: set[str] | None = None): + self.known_agents = known_agents or set() + + def extract_identity(self, data: dict) -> tuple[Optional[str], Optional[str]]: + tool = data.get("tool_name", "") + tool_input = data.get("tool_input") or {} + if not isinstance(tool_input, dict): + tool_input = {} + + current: Optional[str] = None + dispatched: Optional[str] = None + + if tool in MESSAGE_TOOLS: + extracted_current, extracted_dispatched = self._extract_from_message(tool_input) + if extracted_current and not current: + current = extracted_current + if extracted_dispatched and not dispatched: + dispatched = extracted_dispatched + + if tool in DISPATCH_TOOLS and not dispatched: + dispatched = self._extract_from_task(tool_input) + + if not current: + current = self._extract_from_path(data, tool_input) + + if self.known_agents: + if current and current not in self.known_agents: + current = None + if dispatched and dispatched not in self.known_agents: + dispatched = None + + return current, dispatched + + def track(self, state_path: Path, sid: str, data: dict) -> tuple[str, Optional[str]]: + tool = data.get("tool_name", "") + cur_agent, dispatched = self.extract_identity(data) + + def _update(state: dict[str, Any]) -> tuple[str, Optional[str]]: + sess = st.ensure_session(state, sid) + hist = sess.setdefault("agent_history", []) + prev_agent = sess.get("current_agent") or "main" + + if cur_agent and cur_agent != prev_agent: + sess["current_agent"] = cur_agent + hist.append({"ts": time.time(), "agent": cur_agent, "evidence": f"from_role@{tool}"}) + active_agent = cur_agent + else: + active_agent = prev_agent + + if dispatched and dispatched != active_agent: + dis = sess.setdefault("dispatched", {}) + if not isinstance(dis, dict): + dis = {} + sess["dispatched"] = dis + dis[dispatched] = dis.get(dispatched, 0) + 1 + sess["current_agent"] = dispatched + hist.append({"ts": time.time(), "agent": dispatched, "evidence": f"dispatch@{tool}<-{active_agent}"}) + + if len(hist) > 100: + sess["agent_history"] = hist[-100:] + + return active_agent, dispatched + + return st.update_state_locked(state_path, _update) + + def _extract_from_message(self, tool_input: dict) -> tuple[Optional[str], Optional[str]]: + current: Optional[str] = None + dispatched: Optional[str] = None + content = tool_input.get("content") + + if isinstance(content, str) and content.strip().startswith("{"): + try: + payload = json.loads(content) + current = self._extract_str(payload, "from_role") + next_target = payload.get("next_target") or {} + if isinstance(next_target, dict): + dispatched = ( + self._extract_str(next_target, "role_name") + or self._extract_str(next_target, "subagent_name") + ) + except Exception: + pass + elif isinstance(content, dict): + current = self._extract_str(content, "from_role") + next_target = content.get("next_target") or {} + if isinstance(next_target, dict): + dispatched = ( + self._extract_str(next_target, "role_name") + or self._extract_str(next_target, "subagent_name") + ) + + if not dispatched: + recipient = tool_input.get("recipient") + if isinstance(recipient, str): + normalized = recipient.strip().lower() + if normalized and normalized != "main": + dispatched = normalized + + return current, dispatched + + def _extract_from_task(self, tool_input: dict) -> Optional[str]: + sub = ( + # Claude Code 的 Agent 工具（非 CodeBuddy 原生 Task/team_create）用的是 + # subagent_type 字段，不是 subagent_name——不认这个字段会导致 dispatch + # 统计漏掉所有走 Agent 工具派发的场景。 + tool_input.get("subagent_type") + or tool_input.get("subagent_name") + or tool_input.get("name") + or tool_input.get("agent") + or tool_input.get("role") + ) + if isinstance(sub, str) and sub.strip(): + candidate = sub.strip().lower() + if "/" in candidate: + candidate = candidate.rsplit("/", 1)[1].replace(".md", "") + return candidate + return None + + def _extract_from_path(self, data: dict, tool_input: dict) -> Optional[str]: + text = json.dumps(tool_input, ensure_ascii=False) + " " + (data.get("cwd") or "") + match = AGENT_PATH_RE.search(text) + if match: + return match.group(1).lower() + return None + + @staticmethod + def _extract_str(payload: dict, key: str) -> Optional[str]: + value = payload.get(key) + if isinstance(value, str) and value.strip(): + return value.strip().lower() + return None + + +def _config_matches_cwd(config: dict[str, Any], cwd: str) -> bool: + """判断 team 配置中的成员 cwd 是否覆盖当前工作目录。""" + members = config.get("members") + if not isinstance(members, list): + return False + for member in members: + if not isinstance(member, dict): + continue + member_cwd = member.get("cwd") + if isinstance(member_cwd, str) and _paths_equal(member_cwd, cwd): + return True + return False + + +def _paths_equal(a: str, b: str) -> bool: + """按归一化绝对路径语义比较两个路径是否相等。""" + try: + return Path(a).expanduser().resolve() == Path(b).expanduser().resolve() + except Exception: + return os.path.abspath(os.path.expanduser(a)) == os.path.abspath(os.path.expanduser(b)) + + +def _normalize_message(item: Any, mailbox_name: str, mailbox_role: str | None, index: int) -> dict[str, Any] | None: + """把原始 inbox 消息归一化成统一可推断的结构。""" + if not isinstance(item, dict): + return None + + text = item.get("text") + payload = _coerce_payload(text) + summary = item.get("summary") if isinstance(item.get("summary"), str) else "" + from_raw = item.get("from") if isinstance(item.get("from"), str) else None + from_role = normalize_role_name((payload.get("from") if isinstance(payload, dict) else None) or from_raw) + ts = _parse_timestamp(item.get("timestamp")) + + return { + "mailbox_name": mailbox_name, + "mailbox_role": mailbox_role, + "from": from_raw, + "from_role": from_role, + "text": text if isinstance(text, str) else "", + "summary": summary, + "payload": payload, + "timestamp": item.get("timestamp"), + "ts": ts, + "index": index, + "is_shutdown": _is_shutdown_message(payload, summary, text), + } + + +def _extract_next_target(payload: dict[str, Any] | None) -> Optional[str]: + """从 payload 中提取下一跳要派发给的角色。""" + if not isinstance(payload, dict): + return None + next_target = payload.get("next_target") or {} + if not isinstance(next_target, dict): + return None + return normalize_role_name(next_target.get("role_name") or next_target.get("subagent_name")) + + +def _coerce_payload(text: Any) -> dict[str, Any] | None: + """把文本内容尽量解析成 JSON 字典。""" + if isinstance(text, dict): + return text + if not isinstance(text, str): + return None + stripped = text.strip() + if not stripped.startswith("{"): + return None + try: + data = json.loads(stripped) + if isinstance(data, dict): + return data + except Exception: + return None + return None + + +def _is_shutdown_message(payload: dict[str, Any] | None, summary: str, text: Any) -> bool: + """识别一条 inbox 消息是否表示 agent 关闭或退出。""" + if isinstance(payload, dict) and str(payload.get("type") or "").lower() == "shutdown_request": + return True + summary_low = summary.lower() if isinstance(summary, str) else "" + text_low = text.lower() if isinstance(text, str) else "" + return "shutdown" in summary_low or "shutdown request" in text_low + + +def _parse_timestamp(value: Any) -> float: + """把消息时间字段解析成 Unix 时间戳。""" + if not isinstance(value, str) or not value.strip(): + return datetime.now(timezone.utc).timestamp() + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp() + except Exception: + return datetime.now(timezone.utc).timestamp() + + +def _load_json(path: Path) -> Any: + """安全读取 JSON 文件，失败时返回 ``None``。""" + try: + return json.loads(path.read_text("utf-8")) + except Exception: + return None + + +def _safe_int(value: Any) -> int: + """把任意值尽量转成整数，失败时返回 0。""" + try: + return int(value or 0) + except Exception: + return 0 diff --git a/.claude/skills/agent-observability/scripts/core/agentlens/__init__.py b/.claude/skills/agent-observability/scripts/core/agentlens/__init__.py new file mode 100644 index 0000000..2fea21d --- /dev/null +++ b/.claude/skills/agent-observability/scripts/core/agentlens/__init__.py @@ -0,0 +1,9 @@ +"""可选的 AgentLens sink，用于实时镜像 trace。""" +from .runtime import emit_post_step, emit_session_start, emit_session_stop, emit_turn_start + +__all__ = [ + "emit_post_step", + "emit_session_start", + "emit_session_stop", + "emit_turn_start", +] diff --git a/.claude/skills/agent-observability/scripts/core/agentlens/bootstrap.py b/.claude/skills/agent-observability/scripts/core/agentlens/bootstrap.py new file mode 100644 index 0000000..d6fc970 --- /dev/null +++ b/.claude/skills/agent-observability/scripts/core/agentlens/bootstrap.py @@ -0,0 +1,229 @@ +from __future__ import annotations +"""AgentLens 启动辅助层。 + +这一层负责“读入与初始化”： +- 解析项目配置与环境变量 +- 延迟加载可选的 zhiyan 运行时 +- 缓存初始化结果，避免重复 init +- 处理调试落盘与失败状态写回 +""" + +import getpass +import json +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from .. import state as st + +DEFAULT_APP_NAME = "skillhub.codebuddy-hooks" +DEFAULT_BUSINESS_SCENARIO = "codebuddy-hook" + +_RUNTIME: dict[str, Any] | None = None +_INIT_SIGNATURE: tuple[str, str, str] | None = None + + +@dataclass(frozen=True) +class AgentLensConfig: + """从环境变量与 env 文件归一化得到的 AgentLens 配置。""" + endpoint: str | None + api_key: str | None + app_name: str + business_scenario: str + user: str + + @property + def enabled(self) -> bool: + return bool(self.endpoint and self.api_key) + + +def project_root(cwd: str | None = None) -> Path: + """解析用于查找 `.env` 的项目根目录。""" + if cwd: + return Path(cwd).expanduser().resolve() + env_cwd = str(os.environ.get("CODEBUDDY_PROJECT_DIR") or "").strip() + if env_cwd: + return Path(env_cwd).expanduser().resolve() + return Path.cwd().resolve() + + +def read_env_file(path: Path) -> dict[str, str]: + """以轻量方式解析 shell 风格的 env 文件，不做 source。""" + values: dict[str, str] = {} + if not path.is_file(): + return values + try: + lines = path.read_text("utf-8").splitlines() + except Exception: + return values + + for raw_line in lines: + line = raw_line.strip() + if not line or line.startswith("#"): + continue + if line.startswith("export "): + line = line[7:].strip() + if "=" not in line: + continue + key, value = line.split("=", 1) + key = key.strip() + value = value.strip() + if not key: + continue + if value and value[0] == value[-1] and value[0] in {"'", '"'}: + value = value[1:-1] + values[key] = value + return values + + +def load_config(project_root_path: Path) -> AgentLensConfig: + """按优先级加载配置：进程环境变量 > `.env.local` > `.env`。""" + env_file = read_env_file(project_root_path / ".env") + env_local_file = read_env_file(project_root_path / ".env.local") + + def _value(name: str, default: str | None = None) -> str | None: + env_value = os.environ.get(name) + if env_value is not None and str(env_value).strip(): + return str(env_value).strip() + if name in env_local_file and str(env_local_file[name]).strip(): + return str(env_local_file[name]).strip() + if name in env_file and str(env_file[name]).strip(): + return str(env_file[name]).strip() + return default + + user = _value("ZHIYANLLM_USER") + if not user: + user = ( + str(os.environ.get("USER") or "").strip() + or str(os.environ.get("USERNAME") or "").strip() + or getpass.getuser() + ) + + return AgentLensConfig( + endpoint=_value("ZHIYANLLM_API_ENDPOINT"), + api_key=_value("ZHIYANLLM_API_KEY"), + app_name=_value("ZHIYANLLM_APP_NAME", DEFAULT_APP_NAME) or DEFAULT_APP_NAME, + business_scenario=_value("ZHIYANLLM_BUSINESS_SCENARIO", DEFAULT_BUSINESS_SCENARIO) + or DEFAULT_BUSINESS_SCENARIO, + user=user or "unknown", + ) + + +def load_runtime() -> dict[str, Any] | None: + """延迟导入可选的 zhiyan 运行时组件。 + + 即使本地没有安装 AgentLens 依赖，hook 主链路也必须继续工作， + 所以这里返回 ``None``，而不是把导入异常抛出去。 + """ + global _RUNTIME + if _RUNTIME is not None: + return _RUNTIME + try: + from zhiyanllm import Zhiyanllm + from zhiyanllm.opentelemetry.instrumentation.semconv_ai import ( + SpanAttributes as ZhiyanSpanAttributes, + ZhiyanllmSpanKindValues, + ) + from zhiyanllm.tracing.context_manager import get_tracer + from zhiyanllm.tracing.manual import track_llm_call, track_task_server_call + from zhiyanllm.tracing.tracing import TracerWrapper + from opentelemetry.context import attach as otel_attach, detach as otel_detach + except Exception: + _RUNTIME = None + return None + + _RUNTIME = { + "Zhiyanllm": Zhiyanllm, + "track_llm_call": track_llm_call, + "track_task_server_call": track_task_server_call, + "TracerWrapper": TracerWrapper, + "get_tracer": get_tracer, + "ZhiyanSpanAttributes": ZhiyanSpanAttributes, + "ZhiyanllmSpanKindValues": ZhiyanllmSpanKindValues, + "otel_attach": otel_attach, + "otel_detach": otel_detach, + } + return _RUNTIME + + +def ensure_initialized(config: AgentLensConfig) -> dict[str, Any] | None: + """仅当有效配置签名发生变化时才重新初始化 zhiyan。""" + global _INIT_SIGNATURE + if not config.enabled: + return None + runtime = load_runtime() + if runtime is None: + return None + signature = (str(config.endpoint), str(config.api_key), str(config.app_name)) + if _INIT_SIGNATURE == signature: + return runtime + runtime["Zhiyanllm"].init( + app_name=str(config.app_name), + api_endpoint=str(config.endpoint), + api_key=str(config.api_key), + disable_batch=True, + ) + _INIT_SIGNATURE = signature + return runtime + + +def project_root_from_state_path(state_path: Path) -> Path: + """尽力从 `logs/.state.json` 反推出项目根目录。""" + try: + current = state_path.resolve().parent + for parent in [current, *current.parents]: + if parent.name == ".codebuddy": + return parent.parent + return state_path.resolve().parents[4] + except Exception: + return Path.cwd().resolve() + + +def debug_enabled(state_path: Path) -> bool: + """判断是否要把 span 调试信息镜像写入 `agentlens-push-debug.ndjson`。""" + root = project_root_from_state_path(state_path) + env_file = read_env_file(root / ".env") + env_local_file = read_env_file(root / ".env.local") + raw = ( + os.environ.get("AGENTLENS_PUSH_DEBUG") + or env_local_file.get("AGENTLENS_PUSH_DEBUG") + or env_file.get("AGENTLENS_PUSH_DEBUG") + or "" + ) + return str(raw).strip().lower() in {"1", "true", "yes", "on"} + + +def debug_write_span( + state_path: Path | None, + sid: str | None, + payload: dict[str, Any], + *, + sanitizer, +) -> None: + """在开启 AgentLens 调试模式时追加写入清洗后的调试记录。""" + if state_path is None or not sid: + return + try: + if not debug_enabled(state_path): + return + debug_path = state_path.parent / "agentlens-push-debug.ndjson" + record = {"sid": sid, **sanitizer(payload)} + with debug_path.open("a", encoding="utf-8") as fp: + fp.write(json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n") + except Exception: + return + + +def set_failure(state_path: Path, sid: str, message: str) -> None: + """记录最近一次 AgentLens 失败，但不打断 hook 主流程。""" + st.update_agentlens_session( + state_path, + sid, + {"enabled": False, "last_error": str(message)}, + ) + + +def get_session_context(state_path: Path, sid: str) -> dict[str, Any]: + """从共享状态里读取 AgentLens sidecar 的 session payload。""" + return st.load_agentlens_session(state_path, sid) diff --git a/.claude/skills/agent-observability/scripts/core/agentlens/normalize.py b/.claude/skills/agent-observability/scripts/core/agentlens/normalize.py new file mode 100644 index 0000000..2b730f0 --- /dev/null +++ b/.claude/skills/agent-observability/scripts/core/agentlens/normalize.py @@ -0,0 +1,323 @@ +from __future__ import annotations +"""AgentLens 负载清洗层。 + +这一层负责“清洗与整形”： +- 在发送或调试前把任意 payload 转成安全的 JSON 结构 +- 把本地 token 结构归一化成类似 OpenAI 的 usage 结构 +- 从 transcript 中重建轻量的 message / tool-call 上下文 +""" + +import json +import os +from pathlib import Path +from typing import Any + +from .. import state as st +from .bootstrap import AgentLensConfig + + +def sanitize_payload(value: Any) -> Any: + """把任意 Python 值转换成适合调试/打点的 JSON 安全结构。""" + try: + return json.loads(json.dumps(value, ensure_ascii=False, default=str)) + except Exception: + return str(value) + + +def association_properties(config: AgentLensConfig, sid: str, state_path: Path | None = None) -> dict[str, str]: + """构造 trace 级关联属性，并允许按 turn 覆盖业务场景。""" + scenario = config.business_scenario + if state_path: + try: + session_ctx = st.load_agentlens_session(state_path, sid) + turn_scenario = session_ctx.get("_turn_business_scenario") + if isinstance(turn_scenario, str) and turn_scenario.strip(): + scenario = turn_scenario.strip() + except Exception: + pass + return { + "session_id": sid, + "business_scenario": scenario, + "user": config.user, + } + + +def to_openai_usage(tokens: dict[str, Any] | None) -> dict[str, int]: + """把本地 token 结构转换成 zhiyan 期望的 usage 形状。""" + if not isinstance(tokens, dict): + return {} + + def _int(v: Any) -> int: + try: + return int(v or 0) + except Exception: + return 0 + + prompt = _int(tokens.get("input")) + completion = _int(tokens.get("output")) + if prompt <= 0 and completion <= 0: + return {} + return {"prompt_tokens": prompt, "completion_tokens": completion, "total_tokens": prompt + completion} + + +def infer_vendor(model: str | None) -> str: + """根据模型名推断一个粗粒度的 provider/vendor 标签。""" + text = str(model or "").strip().lower() + if not text: + return "unknown" + if "gpt" in text or "openai" in text: + return "openai" + if "claude" in text or "anthropic" in text: + return "anthropic" + if "gemini" in text or "google" in text: + return "google" + if text.startswith("hy") or "hunyuan" in text: + return "tencent" + return text.split("/", 1)[0].split("-", 1)[0] or "unknown" + + +def extract_message_text(content: Any) -> str: + """把嵌套的 transcript message 内容压平成纯文本。""" + if content is None: + return "" + if isinstance(content, str): + return content + if isinstance(content, list): + parts: list[str] = [] + for item in content: + if isinstance(item, dict): + txt = item.get("text") + if isinstance(txt, str): + parts.append(txt) + elif item.get("type") == "tool_use": + parts.append(f"[tool_use {item.get('name', '')}]") + elif item.get("type") == "tool_result": + parts.append(extract_message_text(item.get("content"))) + else: + parts.append(str(item)) + return "\n".join(p for p in parts if p) + if isinstance(content, dict): + return extract_message_text(content.get("text") or content.get("content")) + return str(content) + + +def find_message_id_in_record(obj: Any) -> str | None: + """在 transcript 记录里递归查找稳定的 message 标识。""" + message_id_keys = ("messageId", "responseId", "requestId") + if isinstance(obj, dict): + provider = obj.get("providerData") + if isinstance(provider, dict): + for key in message_id_keys: + value = provider.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + extra = obj.get("extra") + if isinstance(extra, dict): + for key in message_id_keys: + value = extra.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + for key in message_id_keys: + value = obj.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + for value in obj.values(): + found = find_message_id_in_record(value) + if found: + return found + elif isinstance(obj, list): + for value in obj: + found = find_message_id_in_record(value) + if found: + return found + return None + + +def collect_tool_calls_for_message(transcript_path: str, message_id: str) -> list[dict[str, Any]]: + """为指定 assistant message 重建它挂载的 tool-call 摘要。""" + if not transcript_path or not message_id or not os.path.isfile(transcript_path): + return [] + try: + with open(transcript_path, "rb") as fp: + blob = fp.read() + except Exception: + return [] + + tool_calls: list[dict[str, Any]] = [] + for raw_line in blob.splitlines(keepends=True): + if not raw_line.lstrip().startswith(b"{"): + continue + try: + rec = json.loads(raw_line.decode("utf-8", errors="ignore")) + except Exception: + continue + if not isinstance(rec, dict): + continue + if str(rec.get("type") or "").strip().lower() != "function_call": + continue + rec_mid = find_message_id_in_record(rec) + if rec_mid != message_id: + continue + call_id = str(rec.get("callId") or "").strip() + name = str(rec.get("name") or "").strip() + arguments = rec.get("arguments") + entry: dict[str, Any] = {"id": call_id, "name": name} + if arguments is not None: + entry["arguments"] = arguments if isinstance(arguments, str) else json.dumps(arguments, ensure_ascii=False) + tool_calls.append(entry) + return tool_calls + + +def find_tool_usage_event(tool_event: dict[str, Any], usage_events: list[dict[str, Any]]) -> dict[str, Any] | None: + """从 usage 列表里挑出最可能属于当前 tool 事件的那一条。""" + tool_name = str(tool_event.get("tool") or "") + transcript_path = str(tool_event.get("transcript_path") or "") + agent = str(tool_event.get("agent") or "main") + for usage_event in usage_events: + if not isinstance(usage_event, dict): + continue + if str(usage_event.get("agent") or "main") != agent: + continue + if transcript_path and str(usage_event.get("transcript_path") or "") != transcript_path: + continue + if tool_name and str(usage_event.get("tool") or "") not in {tool_name, "model_request"}: + continue + if str(usage_event.get("message_id") or "").strip(): + return usage_event + for usage_event in usage_events: + if not isinstance(usage_event, dict): + continue + if str(usage_event.get("agent") or "main") != agent: + continue + if transcript_path and str(usage_event.get("transcript_path") or "") != transcript_path: + continue + return usage_event + return None + + +def build_llm_io_from_transcript( + *, + transcript_path: str, + source_offset: int, + tokens: dict[str, Any] | None, + model: str | None, + start_offset: int = 0, + current_message_id: str | None = None, +) -> tuple[dict[str, Any], dict[str, Any], str]: + """从 transcript 历史中构造紧凑版的 LLM 输入/输出负载。""" + input_data: dict[str, Any] = {"model": str(model or "unknown"), "messages": []} + output_data: dict[str, Any] = {"choices": [], "usage": to_openai_usage(tokens)} + if not transcript_path or not os.path.isfile(transcript_path): + return input_data, output_data, "" + try: + with open(transcript_path, "rb") as fp: + fp.seek(0) + blob = fp.read(source_offset) if source_offset else fp.read() + except Exception: + return input_data, output_data, "" + + messages: list[dict[str, Any]] = [] + system_prompts: list[str] = [] + cursor = 0 + last_asst_text: str | None = None + + def _truncate(text: str, limit: int = 4000) -> str: + if len(text) <= limit: + return text + return text[:limit] + "...(truncated)" + + for raw_line in blob.splitlines(keepends=True): + cursor += len(raw_line) + if not raw_line.lstrip().startswith(b"{"): + continue + try: + rec = json.loads(raw_line.decode("utf-8", errors="ignore")) + except Exception: + continue + if not isinstance(rec, dict): + continue + + rec_type = str(rec.get("type") or "").strip().lower() + is_incremental = cursor > start_offset + + # 重建 prompt 上下文时要跳过“当前正在发出的 assistant message”， + # 否则同一段回复会同时出现在 input 和 output 两边。 + if current_message_id and is_incremental: + rec_mid = find_message_id_in_record(rec) + if rec_mid == current_message_id: + if rec_type == "message" and rec.get("role") == "assistant": + text = extract_message_text(rec.get("content")) + if text: + last_asst_text = text + continue + + if rec_type == "message": + role = rec.get("role") + text = extract_message_text(rec.get("content")) + if not text: + continue + if role == "system": + system_prompts.append(text) + elif is_incremental: + if role == "user": + messages.append({"role": "user", "content": _truncate(text, 8000)}) + last_asst_text = None + elif role == "assistant": + messages.append({"role": "assistant", "content": _truncate(text, 4000)}) + last_asst_text = text + elif is_incremental: + # 把 function call / result 记录转换成 Chat Completions 风格的 + # assistant/tool message，方便 tracing UI 按对话链路展示。 + if rec_type == "function_call": + name = str(rec.get("name") or "").strip() + call_id = str(rec.get("callId") or "").strip() + arguments = rec.get("arguments", "") + if isinstance(arguments, dict): + arguments = json.dumps(arguments, ensure_ascii=False) + tool_call: dict[str, Any] = {"type": "function", "function": {"name": name}} + if name and arguments: + tool_call["function"]["arguments"] = _truncate(str(arguments), 2000) + if call_id: + tool_call["id"] = call_id + if ( + messages + and messages[-1].get("role") == "assistant" + and "tool_calls" not in messages[-1] + and messages[-1].get("content") + ): + messages[-1]["tool_calls"] = [tool_call] + else: + msg: dict[str, Any] = {"role": "assistant", "content": ""} + msg["tool_calls"] = [tool_call] + messages.append(msg) + elif rec_type == "function_call_result": + call_id = str(rec.get("callId") or "").strip() + provider = rec.get("providerData", {}) + result_content = "" + if isinstance(provider, dict): + tool_result = provider.get("toolResult", {}) + if isinstance(tool_result, dict): + result_content = extract_message_text(tool_result.get("content")) + if not result_content: + output = rec.get("output") + result_content = str(output.get("text", "")) if isinstance(output, dict) else "" + tool_msg: dict[str, Any] = {"role": "tool", "content": _truncate(result_content, 2000)} + if call_id: + tool_msg["tool_call_id"] = call_id + messages.append(tool_msg) + + final_messages: list[dict[str, Any]] = [] + for prompt in system_prompts[-2:]: + if prompt: + final_messages.append({"role": "system", "content": _truncate(prompt, 2000)}) + final_messages.extend(messages) + input_data["messages"] = final_messages + + if last_asst_text is not None: + output_data["choices"] = [{ + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": last_asst_text[:8000]}, + }] + + return input_data, output_data, "" diff --git a/.claude/skills/agent-observability/scripts/core/agentlens/runtime.py b/.claude/skills/agent-observability/scripts/core/agentlens/runtime.py new file mode 100644 index 0000000..321f45a --- /dev/null +++ b/.claude/skills/agent-observability/scripts/core/agentlens/runtime.py @@ -0,0 +1,761 @@ +"""AgentLens 运行时编排层。 + +这一层保留对外暴露的 4 个入口： +- `emit_session_start` +- `emit_turn_start` +- `emit_post_step` +- `emit_session_stop` + +它本身不负责底层清洗或 carrier 生成，而是把调用编排到 +`bootstrap / normalize / tracing` 三层上。 +""" +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from opentelemetry import trace as _otel_trace + +from .. import state as st +from .bootstrap import ( + debug_write_span as _debug_write_span, + ensure_initialized as _ensure_initialized, + get_session_context as _get_session_context, + load_config, + project_root as _project_root, + set_failure as _set_failure, +) +from .normalize import ( + association_properties as _association_properties, + build_llm_io_from_transcript as _build_llm_io_from_transcript, + find_tool_usage_event as _find_tool_usage_event, + infer_vendor as _infer_vendor, + sanitize_payload as _sanitize_payload, + to_openai_usage as _to_openai_usage, +) +from .tracing import ( + annotate as _annotate, + emit_assistant_span as _emit_assistant_span, + emit_tool_span as _emit_tool_span, + generate_subagent_carrier as _generate_subagent_carrier, + generate_turn_carrier as _generate_turn_carrier, + resolve_agent_carrier as _resolve_agent_carrier, + resolve_step_carrier as _resolve_step_carrier, + set_agent_aggregate_on_span as _set_agent_aggregate_on_span, + span_context_ids as _span_context_ids, + traceparent_parts as _traceparent_parts, +) + + +def emit_session_start( + *, + state_path: Path, + sid: str, + cwd: str, + agent: str = "main", +) -> None: + """初始化 v2 版本的 `_agentlens` session payload。 + + 在 turn-centric 模型下，这里**不再**生成 trace_id。 + 它只负责记录 session 级元数据（如 `enabled`、`session_id`、`app_name`）， + 并清理上一次运行遗留的 carrier / error 状态。 + 真正的新 trace 会在第一次 `UserPromptSubmit` 时由 `emit_turn_start` 打开。 + """ + _ = agent + config = load_config(_project_root(cwd)) + # 注意：这里不要清空 current_turn / turn_history。 + # 同一个 sid 下 SessionStart 可能多次触发（例如 subagent 启动、IDE 刷新、 + # workspace 切换）。如果这里清空，会把进行中的 turn 擦掉，后续 PostToolUse + # 会全部看到 no_active_turn，最终把这一轮观测链路打断。 + base_updates: dict[str, Any] = { + "session_id": sid, + "app_name": config.app_name, + } + if not config.enabled: + base_updates["enabled"] = False + st.update_agentlens_session( + state_path, + sid, + base_updates, + clear_keys=["carrier", "last_error"], + ) + return + + try: + runtime = _ensure_initialized(config) + if runtime is None: + _set_failure(state_path, sid, "zhiyanllm runtime unavailable") + return + base_updates["enabled"] = True + st.update_agentlens_session( + state_path, + sid, + base_updates, + clear_keys=["carrier", "last_error"], + ) + except Exception as exc: + _set_failure(state_path, sid, str(exc)) + + +def emit_turn_start( + *, + state_path: Path, + sid: str, + cwd: str, + turn_id: str, + prompt_meta: dict[str, Any] | None = None, + business_scenario: str | None = None, +) -> None: + """打开一个新的 turn，生成新的 trace_id 并写入 `_agentlens.current_turn`。 + + 这是当前实现里**唯一**允许生成新 trace_id 的地方。 + `emit_post_step` / `emit_session_stop` 都不会重新造 carrier； + 找不到时只会优雅降级，不会私自开新链路。 + """ + _ = prompt_meta + config = load_config(_project_root(cwd)) + if not config.enabled: + return + + try: + runtime = _ensure_initialized(config) + if runtime is None: + _set_failure(state_path, sid, "zhiyanllm runtime unavailable") + return + # 先把 per-turn business scenario 写进 state， + # 这样 `_generate_turn_carrier -> _association_properties` + # 才能在生成 trace 时把这次 turn 的业务场景烘焙进去。 + if business_scenario: + st.update_agentlens_session( + state_path, + sid, + {"enabled": True, "session_id": sid, "app_name": config.app_name, "_turn_business_scenario": business_scenario}, + clear_keys=["last_error"], + ) + else: + st.update_agentlens_session( + state_path, + sid, + {"enabled": True, "session_id": sid, "app_name": config.app_name}, + clear_keys=["last_error", "_turn_business_scenario"], + ) + carrier = _generate_turn_carrier(runtime, config=config, sid=sid, turn_id=turn_id, state_path=state_path) + st.begin_turn(state_path, sid, turn_id, carrier) + except Exception as exc: + _set_failure(state_path, sid, str(exc)) + + +def emit_post_step( + *, + state_path: Path, + sid: str, + tool_event: dict[str, Any] | None, + usage_events: list[dict[str, Any]], +) -> None: + """发出扁平化的 LLM span，只保留最小 TASK 包装层来传播 carrier 上下文。 + + 面向 turn 的 v2 约束： + - 活跃的 ``current_turn.carrier`` 是唯一的 trace 上下文来源。 + - 如果不存在 ``current_turn``（例如 UserPromptSubmit hook 还没触发）， + 这里只会记录 ``last_error`` 并返回，不会私自生成新的 trace_id。 + - subagent（``active_agent != "main"``）会拿到当前 turn 下的 + ``invoke_agent`` 子 span，后续 chat span 都挂到这个子 span 下， + 以保持因果链路连续。 + """ + if not isinstance(tool_event, dict) and not usage_events: + return + + cwd_hint = "" + if isinstance(tool_event, dict): + cwd_hint = str(tool_event.get("cwd") or "") + config = load_config(_project_root(cwd_hint)) + if not config.enabled: + return + + session_ctx = _get_session_context(state_path, sid) + # 快路径：尊重 SessionStart 阶段已经写入的 enabled 标记。 + if session_ctx.get("enabled") is False: + return + + current_turn = session_ctx.get("current_turn") + if not isinstance(current_turn, dict): + _set_failure(state_path, sid, "no_active_turn") + return + parent_carrier = current_turn.get("carrier") + if not isinstance(parent_carrier, dict) or not parent_carrier.get("traceparent"): + _set_failure(state_path, sid, "no_active_turn_carrier") + return + + try: + runtime = _ensure_initialized(config) + if runtime is None: + _set_failure(state_path, sid, "zhiyanllm runtime unavailable") + return + + tool_name = str((tool_event or {}).get("tool") or "unknown_tool") + active_agent = str((tool_event or {}).get("agent") or "main") + transcript_path = str((tool_event or {}).get("transcript_path") or "") + skills = (tool_event or {}).get("skill") or [] + rules = (tool_event or {}).get("rule") or [] + + # 解析这次事件真正要用的 carrier。 + # main agent 直接使用 turn 根 carrier；subagent 则使用按角色拆分的 + # 子 span carrier，并在当前 turn 下做幂等注册。 + carrier = dict(parent_carrier) + if active_agent and active_agent != "main": + existing = None + subs = current_turn.get("subagent_spans") + if isinstance(subs, dict): + rec = subs.get(active_agent) + if isinstance(rec, dict) and isinstance(rec.get("carrier"), dict): + existing = rec["carrier"] + if isinstance(existing, dict) and existing.get("traceparent"): + carrier = dict(existing) + else: + # 原子化的 get-or-create：只有确认 subagent span 不存在时， + # 才会在写锁内部调用 carrier_factory。 + # 这样可以避免并发 hook 进程之间的 TOCTOU 竞争，产生孤儿 + # invoke_agent.TASK span。 + def _factory() -> dict[str, str]: + return _generate_subagent_carrier( + runtime, parent_carrier=parent_carrier, role=active_agent + ) + + registered = st.upsert_subagent_span_atomic( + state_path, sid, active_agent, carrier_factory=_factory + ) + if isinstance(registered, dict) and registered.get("traceparent"): + carrier = dict(registered) + runtime["Zhiyanllm"].set_association_properties(_association_properties(config, sid, state_path)) + # 静默挂载 carrier 上下文（不额外创建 TASK span），让内部 span + # 继承正确的 trace_id / parent_id。 + _ctx = runtime["TracerWrapper"].extract_context(carrier) + _token = runtime["otel_attach"](_ctx) + try: + preferred_tool_message_id = str((tool_event or {}).get("message_id") or "").strip() or None + _assistant_emitted_mids: set[str] = set() + pre_bumped_steps: set[tuple[str, str]] = set() + committed_steps: set[tuple[str, str]] = set() + + if isinstance(tool_event, dict): + if preferred_tool_message_id and st.get_step_span_carrier( + state_path, sid, active_agent, preferred_tool_message_id + ) is None: + pre_bumped_steps.add((active_agent, preferred_tool_message_id)) + for usage_event in usage_events: + event_agent = str(usage_event.get("agent") or active_agent or "main") + event_message_id = str(usage_event.get("message_id") or "").strip() or None + if event_message_id: + step_key = (event_agent, event_message_id) + if ( + step_key not in pre_bumped_steps + and st.get_step_span_carrier(state_path, sid, event_agent, event_message_id) is None + ): + pre_bumped_steps.add(step_key) + if not usage_events and isinstance(tool_event, dict): + tool_parent_carrier = _resolve_agent_carrier( + state_path=state_path, + sid=sid, + runtime=runtime, + parent_carrier=carrier, + agent=active_agent, + ) + if preferred_tool_message_id: + step_carrier = _resolve_step_carrier( + state_path=state_path, + sid=sid, + runtime=runtime, + parent_carrier=tool_parent_carrier, + agent=active_agent, + message_id=preferred_tool_message_id, + transcript_path=transcript_path, + ) + if isinstance(step_carrier, dict) and step_carrier.get("traceparent"): + tool_parent_carrier = dict(step_carrier) + _emit_tool_span( + runtime, + carrier=tool_parent_carrier, + tool_name=tool_name, + active_agent=active_agent, + skills=skills, + rules=rules, + duration_ms=tool_event.get("ms") if isinstance(tool_event.get("ms"), (int, float)) else None, + tool_details=tool_event.get("tool_details") if isinstance(tool_event.get("tool_details"), dict) else None, + message_id=preferred_tool_message_id, + step_grouping="message_id" if preferred_tool_message_id else "fallback", + state_path=state_path, + sid=sid, + ) + st.bump_agent_aggregate( + state_path, + sid, + active_agent, + tool_event=tool_event, + ) + if ( + preferred_tool_message_id + and (active_agent, preferred_tool_message_id) in pre_bumped_steps + and (active_agent, preferred_tool_message_id) not in committed_steps + ): + st.bump_agent_aggregate( + state_path, + sid, + active_agent, + step_created=True, + ) + committed_steps.add((active_agent, preferred_tool_message_id)) + st.update_agentlens_session(state_path, sid, {"enabled": True}, clear_keys=["last_error"]) + return + + matched_usage = _find_tool_usage_event(tool_event, usage_events) if isinstance(tool_event, dict) else None + tool_message_id = preferred_tool_message_id or str((matched_usage or {}).get("message_id") or "").strip() or None + if isinstance(tool_event, dict): + # 先构造 assistant 预览，再打开 agent/step 父 span。 + # 这样即便 transcript 重建失败，也不会留下空的分组 TASK。 + _assistant_preview_content: str | None = None + _assistant_preview_transcript = transcript_path + _assistant_preview_model = "" + if tool_message_id: + for _ue in usage_events: + if str(_ue.get("message_id") or "").strip() != tool_message_id: + continue + _ue_transcript = str(_ue.get("transcript_path") or transcript_path) + _ue_offset = int(_ue.get("source_offset") or 0) + _, _llm_out_pre, _ = _build_llm_io_from_transcript( + transcript_path=_ue_transcript, + source_offset=_ue_offset, + tokens=_ue.get("tokens"), + model=str(_ue.get("model") or ""), + ) + _pre_content = "" + if _llm_out_pre.get("choices"): + _pre_msg = _llm_out_pre["choices"][0].get("message") or {} + _pre_content = str(_pre_msg.get("content") or "") + if _pre_content and _pre_content.strip(): + _assistant_preview_content = _pre_content + _assistant_preview_transcript = _ue_transcript + _assistant_preview_model = str(_ue.get("model") or "") + break + + tool_parent_carrier = _resolve_agent_carrier( + state_path=state_path, + sid=sid, + runtime=runtime, + parent_carrier=carrier, + agent=active_agent, + ) + if tool_message_id: + step_carrier = _resolve_step_carrier( + state_path=state_path, + sid=sid, + runtime=runtime, + parent_carrier=tool_parent_carrier, + agent=active_agent, + message_id=tool_message_id, + transcript_path=transcript_path, + ) + if isinstance(step_carrier, dict) and step_carrier.get("traceparent"): + tool_parent_carrier = dict(step_carrier) + if _assistant_preview_content and tool_message_id: + _emit_assistant_span( + runtime, + carrier=tool_parent_carrier, + agent=active_agent, + message_id=tool_message_id, + assistant_text=_assistant_preview_content, + model=_assistant_preview_model, + transcript_path=_assistant_preview_transcript, + state_path=state_path, + sid=sid, + ) + _assistant_emitted_mids.add(tool_message_id) + + _emit_tool_span( + runtime, + carrier=tool_parent_carrier, + tool_name=tool_name, + active_agent=active_agent, + skills=skills, + rules=rules, + duration_ms=tool_event.get("ms") if isinstance(tool_event.get("ms"), (int, float)) else None, + tool_details=tool_event.get("tool_details") if isinstance(tool_event.get("tool_details"), dict) else None, + message_id=tool_message_id, + step_grouping="message_id" if tool_message_id else "fallback", + state_path=state_path, + sid=sid, + ) + st.bump_agent_aggregate( + state_path, + sid, + active_agent, + tool_event=tool_event, + ) + if tool_message_id and (active_agent, tool_message_id) in pre_bumped_steps: + step_key = (active_agent, tool_message_id) + if step_key not in committed_steps: + st.bump_agent_aggregate( + state_path, + sid, + active_agent, + step_created=True, + ) + committed_steps.add(step_key) + + _prev_llm_offsets: dict[str, int] = {} + for usage_event in usage_events: + model = str(usage_event.get("model") or "") + ev_transcript = str(usage_event.get("transcript_path") or transcript_path) + ev_offset = int(usage_event.get("source_offset") or 0) + raw_tokens = usage_event.get("tokens") or {} + event_message_id = str(usage_event.get("message_id") or "").strip() or None + event_agent = str(usage_event.get("agent") or active_agent or "main") + start_offset = _prev_llm_offsets.get(ev_transcript) or st.get_last_llm_offset( + st.load_state(state_path), sid, ev_transcript + ) + llm_input, llm_output, _turn_title = _build_llm_io_from_transcript( + transcript_path=ev_transcript, + source_offset=ev_offset, + start_offset=int(start_offset or 0), + current_message_id=event_message_id, + tokens=raw_tokens, + model=model or None, + ) + _prev_llm_offsets[ev_transcript] = ev_offset + # 原子写回 state，供跨进程增量跟踪复用 + def _update_llm_offset(state: dict[str, Any], _ev=ev_transcript, _off=ev_offset) -> None: + st.set_last_llm_offset(state, sid, _off, _ev) + st.update_state_locked(state_path, _update_llm_offset) + # --- 把 skill/rule/agent 作为独立消息标签注入 input_data --- + _meta_messages: list[dict[str, str]] = [] + _ev_agent = str(usage_event.get("agent") or active_agent) + if _ev_agent and _ev_agent != "main": + _meta_messages.append({"role": "agent", "content": _ev_agent}) + if skills: + _meta_messages.append({"role": "skill", "content": ", ".join(skills)}) + if rules: + _meta_messages.append({"role": "rule", "content": ", ".join(rules)}) + if _meta_messages: + llm_input.setdefault("messages", []) + for msg in reversed(_meta_messages): + llm_input["messages"].insert(0, msg) + # --- 构造 usage --- + llm_output["usage"] = _to_openai_usage(raw_tokens) + # --- 只附带成本细节来构造 output 内容 --- + _cost_info = { + "input_tokens": raw_tokens.get("input"), + "output_tokens": raw_tokens.get("output"), + "cache_read": raw_tokens.get("cache_read"), + "cache_creation": raw_tokens.get("cache_creation"), + "total_tokens": raw_tokens.get("total"), + "cost_usd": usage_event.get("cost_usd"), + } + _cost_info = {k: v for k, v in _cost_info.items() if v is not None} + _existing_content = "" + if llm_output.get("choices"): + _msg = llm_output["choices"][0].get("message") or {} + _existing_content = str(_msg.get("content") or "") + _output_content = _existing_content + if _cost_info: + _cost_line = " | ".join(f"{k}: {v}" for k, v in _cost_info.items()) + _output_content = f"[{_cost_line}]\n{_existing_content}" if _existing_content else f"[{_cost_line}]" + llm_output["choices"] = [{ + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": _output_content}, + }] + event_parent_carrier = carrier + if event_message_id: + event_parent_carrier = _resolve_agent_carrier( + state_path=state_path, + sid=sid, + runtime=runtime, + parent_carrier=carrier, + agent=event_agent, + ) + step_carrier = _resolve_step_carrier( + state_path=state_path, + sid=sid, + runtime=runtime, + parent_carrier=event_parent_carrier, + agent=event_agent, + message_id=event_message_id, + transcript_path=ev_transcript, + ) + if isinstance(step_carrier, dict) and step_carrier.get("traceparent"): + event_parent_carrier = dict(step_carrier) + ev_ctx = runtime["TracerWrapper"].extract_context(event_parent_carrier) + ev_token = runtime["otel_attach"](ev_ctx) + # 先累计聚合值，再发 LLM span，这样 span 上带的是最新累计属性。 + st.bump_agent_aggregate( + state_path, + sid, + event_agent, + usage_event=usage_event, + ) + _current_agg = st.get_subagent_aggregate(state_path, sid, event_agent) + try: + with runtime["track_llm_call"](_infer_vendor(model), "model_request") as llm_span: + # track_llm_call 已经设置了 gen_ai.span.kind/system/operation.name。 + # 下方 _annotate 会通过 handle_llm_response 设置 gen_ai.usage.*。 + # LLMSpan 包装层没有 set_attribute，所以这里改用原生 span 写 ID。 + _native_span = _otel_trace.get_current_span() + _set_agent_aggregate_on_span(_native_span, _current_agg) + span_ids = _span_context_ids(_native_span) + parent_parts = _traceparent_parts(event_parent_carrier) + _debug_write_span( + state_path, + sid, + { + "event": "span_emit", + "span_kind": "llm", + "span_name": "model_request", + "model": model, + "message_id": str(event_message_id or ""), + "step_grouping": "message_id" if event_message_id else "fallback", + "agent": event_agent, + "trace_id": span_ids.get("trace_id", parent_parts.get("trace_id", "")), + "span_id": span_ids.get("span_id", ""), + "parent_span_id": parent_parts.get("span_id", ""), + "parent_traceparent": parent_parts.get("traceparent", ""), + "transcript_path": ev_transcript, + "source_offset": ev_offset, + }, + sanitizer=_sanitize_payload, + ) + _annotate( + runtime, + span=llm_span, + input_data=llm_input, + output_data=llm_output, + tags={ + "cost_usd": str(usage_event.get("cost_usd") or "-"), + "input_tokens": str(raw_tokens.get("input", "-")), + "output_tokens": str(raw_tokens.get("output", "-")), + "cache_read": str(raw_tokens.get("cache_read", "-")), + "cache_creation": str(raw_tokens.get("cache_creation", "-")), + "total_tokens": str(raw_tokens.get("total", "-")), + "message_id": str(event_message_id or ""), + "step_grouping": "message_id" if event_message_id else "fallback", + }, + ) + finally: + runtime["otel_detach"](ev_token) + if event_message_id and (event_agent, event_message_id) in pre_bumped_steps: + step_key = (event_agent, event_message_id) + if step_key not in committed_steps: + st.bump_agent_aggregate( + state_path, + sid, + event_agent, + step_created=True, + ) + committed_steps.add(step_key) + if _existing_content and _existing_content.strip() and event_message_id not in _assistant_emitted_mids: + _emit_assistant_span( + runtime, + carrier=event_parent_carrier, + agent=event_agent, + message_id=event_message_id, + assistant_text=_existing_content, + model=model, + transcript_path=ev_transcript, + state_path=state_path, + sid=sid, + ) + finally: + runtime["otel_detach"](_token) + st.update_agentlens_session(state_path, sid, {"enabled": True}, clear_keys=["last_error"]) + except Exception as exc: + _set_failure(state_path, sid, str(exc)) + + +def emit_session_stop( + *, + state_path: Path, + sid: str, + cwd: str, + active_agent: str, + transcript_path: str, + stop_events: list[dict[str, Any]], +) -> None: + """在 session stop 阶段发出扁平化 LLM span，只保留最小 TASK 包装层传播上下文。""" + config = load_config(_project_root(cwd)) + if not config.enabled: + return + + session_ctx = _get_session_context(state_path, sid) + if session_ctx.get("enabled") is False: + return + + current_turn = session_ctx.get("current_turn") + if not isinstance(current_turn, dict): + _set_failure(state_path, sid, "no_active_turn") + return + carrier = current_turn.get("carrier") + if not isinstance(carrier, dict) or not carrier.get("traceparent"): + _set_failure(state_path, sid, "no_active_turn_carrier") + return + + try: + runtime = _ensure_initialized(config) + if runtime is None: + _set_failure(state_path, sid, "zhiyanllm runtime unavailable") + return + + runtime["Zhiyanllm"].set_association_properties(_association_properties(config, sid, state_path)) + if not stop_events: + return + # 静默挂载 carrier 上下文（不额外创建 TASK span）。 + _ctx = runtime["TracerWrapper"].extract_context(carrier) + _token = runtime["otel_attach"](_ctx) + try: + _prev_stop_offsets: dict[str, int] = {} + for stop_event in stop_events: + model = str(stop_event.get("model") or "") + ev_transcript = str(stop_event.get("transcript_path") or transcript_path) + ev_offset = int(stop_event.get("source_offset") or 0) + raw_tokens = stop_event.get("tokens") or {} + event_message_id = str(stop_event.get("message_id") or "").strip() or None + event_agent = str(stop_event.get("agent") or active_agent or "main") + start_offset = _prev_stop_offsets.get(ev_transcript) or st.get_last_llm_offset( + st.load_state(state_path), sid, ev_transcript + ) + llm_input, llm_output, _turn_title = _build_llm_io_from_transcript( + transcript_path=ev_transcript, + source_offset=ev_offset, + start_offset=int(start_offset or 0), + current_message_id=event_message_id, + tokens=raw_tokens, + model=model or None, + ) + _prev_stop_offsets[ev_transcript] = ev_offset + # 原子写回 state，供跨进程增量跟踪复用 + def _update_llm_offset_stop(state: dict[str, Any], _ev=ev_transcript, _off=ev_offset) -> None: + st.set_last_llm_offset(state, sid, _off, _ev) + st.update_state_locked(state_path, _update_llm_offset_stop) + # --- 把 agent 作为独立消息标签注入 input_data --- + _ev_agent = str(stop_event.get("agent") or active_agent) + if _ev_agent and _ev_agent != "main": + llm_input.setdefault("messages", []).insert(0, {"role": "agent", "content": _ev_agent}) + # --- 构造 usage --- + llm_output["usage"] = _to_openai_usage(raw_tokens) + # --- 构造带上下文和成本细节的 output 内容 --- + _cost_info = { + "input_tokens": raw_tokens.get("input"), + "output_tokens": raw_tokens.get("output"), + "cache_read": raw_tokens.get("cache_read"), + "cache_creation": raw_tokens.get("cache_creation"), + "total_tokens": raw_tokens.get("total"), + "cost_usd": stop_event.get("cost_usd"), + "cost_session_usd": stop_event.get("cost_session_usd"), + } + _cost_info = {k: v for k, v in _cost_info.items() if v is not None} + _existing_content = "" + if llm_output.get("choices"): + _msg = llm_output["choices"][0].get("message") or {} + _existing_content = str(_msg.get("content") or "") + _output_content = _existing_content + if _cost_info: + _cost_line = " | ".join(f"{k}: {v}" for k, v in _cost_info.items()) + _output_content = f"[{_cost_line}]\n{_existing_content}" if _existing_content else f"[{_cost_line}]" + llm_output["choices"] = [{ + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": _output_content}, + }] + event_parent_carrier = carrier + # 先解析 agent carrier，让 step 挂在 agent 的 react_agent span 下， + # 而不是直接挂在 turn 根 span 下。 + if event_agent and event_agent != "main": + agent_carrier = _resolve_agent_carrier( + state_path=state_path, + sid=sid, + runtime=runtime, + parent_carrier=carrier, + agent=event_agent, + ) + if isinstance(agent_carrier, dict) and agent_carrier.get("traceparent"): + event_parent_carrier = dict(agent_carrier) + if event_message_id: + step_carrier = _resolve_step_carrier( + state_path=state_path, + sid=sid, + runtime=runtime, + parent_carrier=event_parent_carrier, + agent=event_agent, + message_id=event_message_id, + transcript_path=ev_transcript, + ) + if isinstance(step_carrier, dict) and step_carrier.get("traceparent"): + event_parent_carrier = dict(step_carrier) + ev_ctx = runtime["TracerWrapper"].extract_context(event_parent_carrier) + ev_token = runtime["otel_attach"](ev_ctx) + # 这里不要再 bump，emit_post_step 已经累计过这条 usage_event。 + # SessionStop 会为同一次调用补发一个重复的 LLM span，再累加就会双算。 + # 因此这里只读取当前聚合值。 + _current_agg = st.get_subagent_aggregate(state_path, sid, event_agent) + try: + with runtime["track_llm_call"](_infer_vendor(model), "model_request") as llm_span: + # track_llm_call 已经设置了 gen_ai.span.kind/system/operation.name。 + # 下方 _annotate 会通过 handle_llm_response 设置 gen_ai.usage.*。 + # LLMSpan 包装层没有 set_attribute，所以这里改用原生 span 写 ID。 + _native_span = _otel_trace.get_current_span() + _set_agent_aggregate_on_span(_native_span, _current_agg) + span_ids = _span_context_ids(_native_span) + parent_parts = _traceparent_parts(event_parent_carrier) + _debug_write_span( + state_path, + sid, + { + "event": "span_emit", + "span_kind": "llm", + "span_name": "model_request", + "model": model, + "message_id": str(event_message_id or ""), + "step_grouping": "message_id" if event_message_id else "fallback", + "agent": event_agent, + "trace_id": span_ids.get("trace_id", parent_parts.get("trace_id", "")), + "span_id": span_ids.get("span_id", ""), + "parent_span_id": parent_parts.get("span_id", ""), + "parent_traceparent": parent_parts.get("traceparent", ""), + "transcript_path": ev_transcript, + "source_offset": ev_offset, + }, + sanitizer=_sanitize_payload, + ) + _annotate( + runtime, + span=llm_span, + input_data=llm_input, + output_data=llm_output, + tags={ + "cost_usd": str(stop_event.get("cost_usd") or "-"), + "cost_session_usd": str(stop_event.get("cost_session_usd") or "-"), + "input_tokens": str(raw_tokens.get("input", "-")), + "output_tokens": str(raw_tokens.get("output", "-")), + "cache_read": str(raw_tokens.get("cache_read", "-")), + "cache_creation": str(raw_tokens.get("cache_creation", "-")), + "total_tokens": str(raw_tokens.get("total", "-")), + "message_id": str(event_message_id or ""), + "step_grouping": "message_id" if event_message_id else "fallback", + }, + ) + finally: + runtime["otel_detach"](ev_token) + if _existing_content and _existing_content.strip(): + _emit_assistant_span( + runtime, + carrier=event_parent_carrier, + agent=event_agent, + message_id=event_message_id, + assistant_text=_existing_content, + model=model, + transcript_path=ev_transcript, + state_path=state_path, + sid=sid, + ) + finally: + runtime["otel_detach"](_token) + st.update_agentlens_session(state_path, sid, {"enabled": True}, clear_keys=["last_error"]) + except Exception as exc: + _set_failure(state_path, sid, str(exc)) diff --git a/.claude/skills/agent-observability/scripts/core/agentlens/tracing.py b/.claude/skills/agent-observability/scripts/core/agentlens/tracing.py new file mode 100644 index 0000000..afb678a --- /dev/null +++ b/.claude/skills/agent-observability/scripts/core/agentlens/tracing.py @@ -0,0 +1,504 @@ +from __future__ import annotations +"""AgentLens trace 拓扑层。 + +这一层负责“trace 结构本身”： +- 创建或派生 carrier +- 生成 agent / step / assistant / tool span +- 把后续聚合需要的 span 元数据写回 state +""" + +import json +import secrets +from pathlib import Path +from typing import Any + +from .. import state as st +from .bootstrap import AgentLensConfig, debug_write_span +from .normalize import association_properties, collect_tool_calls_for_message, sanitize_payload + + +def annotate(runtime: dict[str, Any], *, span: Any | None = None, input_data: Any = None, output_data: Any = None, tags: dict[str, Any] | None = None) -> None: + """在调用 zhiyan annotate 前，先统一清洗 payload。""" + _ = span + runtime["Zhiyanllm"].annotate( + input_data=sanitize_payload(input_data), + output_data=sanitize_payload(output_data), + tags=sanitize_payload(tags) if tags else None, + ) + + +def traceparent_parts(carrier: dict[str, str] | None) -> dict[str, str]: + """把 W3C `traceparent` 拆成调试日志里常用的几个字段。""" + traceparent = str((carrier or {}).get("traceparent") or "").strip() + parts = traceparent.split("-") + if len(parts) >= 4: + return { + "traceparent": traceparent, + "trace_id": parts[1], + "span_id": parts[2], + "flags": parts[3], + } + return {"traceparent": traceparent} + + +def span_context_ids(span: Any) -> dict[str, str]: + """从 OTel span 对象中提取十六进制的 trace/span id。""" + try: + ctx = span.get_span_context() + return { + "trace_id": f"{int(ctx.trace_id):032x}", + "span_id": f"{int(ctx.span_id):016x}", + } + except Exception: + return {} + + +def ensure_traceparent(carrier: dict[str, str]) -> None: + """当上游没有成功注入时，补一个最小可用的 `traceparent`。""" + if str(carrier.get("traceparent") or "").strip(): + return + trace_id = secrets.token_hex(16) + parent_id = secrets.token_hex(8) + carrier["traceparent"] = f"00-{trace_id}-{parent_id}-01" + + +def carrier_from_span(parent_carrier: dict[str, str], span: Any) -> dict[str, str]: + """基于父 carrier 和当前 span 生成一个仍在同一 trace 上的子 carrier。""" + try: + span_ctx = span.get_span_context() + trace_id = f"{int(span_ctx.trace_id):032x}" + span_id = f"{int(span_ctx.span_id):016x}" + except Exception: + return {} + parent_tp = str((parent_carrier or {}).get("traceparent") or "").strip() + flags = "01" + if parent_tp: + parts = parent_tp.split("-") + if len(parts) >= 4 and parts[3]: + flags = parts[3] + child_carrier: dict[str, str] = dict(parent_carrier or {}) + child_carrier["traceparent"] = f"00-{trace_id}-{span_id}-{flags}" + return child_carrier + + +def carrier_from_task_span(parent_carrier: dict[str, str], task_span: Any) -> dict[str, str]: + """兼容包装过的 task span 和原始 OTel span 两种形态。""" + span = getattr(task_span, "_span", None) + if span is None and hasattr(task_span, "get_span_context"): + span = task_span + if span is None: + return {} + return carrier_from_span(parent_carrier, span) + + +def zhiyan_attr(runtime: dict[str, Any], name: str, fallback: str) -> str: + """防御式读取 zhiyan 语义约定常量。""" + return str(getattr(runtime.get("ZhiyanSpanAttributes"), name, fallback)) + + +def generate_turn_carrier( + runtime: dict[str, Any], + *, + config: AgentLensConfig, + sid: str, + turn_id: str, + state_path: Path | None = None, +) -> dict[str, str]: + """为一个用户 turn 生成根 carrier。""" + from opentelemetry import trace + + carrier: dict[str, str] = {} + span = trace.get_tracer(__name__).start_span(f"turn.{sid}.{turn_id}") + try: + runtime["Zhiyanllm"].set_association_properties(association_properties(config, sid, state_path)) + runtime["Zhiyanllm"].inject_context(carrier) + finally: + span.end() + ensure_traceparent(carrier) + return carrier + + +def generate_subagent_carrier(runtime: dict[str, Any], *, parent_carrier: dict[str, str], role: str) -> dict[str, str]: + """在当前 turn 下面创建 `invoke_agent` 子 carrier。""" + try: + with runtime["track_task_server_call"]("invoke_agent", carrier=parent_carrier) as task_span: + annotate(runtime, input_data={"agent": role}, output_data={"result": "dispatched"}, tags={"agent": role}) + child_carrier = carrier_from_task_span(parent_carrier, task_span) + if child_carrier: + return child_carrier + except Exception: + return {} + return {} + + +def generate_step_carrier( + runtime: dict[str, Any], + *, + parent_carrier: dict[str, str], + agent: str, + message_id: str, + transcript_path: str, +) -> dict[str, str]: + """创建用于归并同一条 message 工作的 synthetic step span。""" + try: + with runtime["track_task_server_call"]("react_step", carrier=parent_carrier) as task_span: + annotate( + runtime, + input_data={"agent": agent, "message_id": message_id}, + output_data={"result": "grouped"}, + tags={"message_id": message_id, "agent": agent, "transcript_path": transcript_path or ""}, + ) + child_carrier = carrier_from_task_span(parent_carrier, task_span) + if child_carrier: + return child_carrier + except Exception: + return {} + return {} + + +def generate_agent_carrier(runtime: dict[str, Any], *, parent_carrier: dict[str, str], agent: str) -> dict[str, str]: + """在当前 turn 下创建稳定的 per-agent 聚合 span。""" + try: + with runtime["track_task_server_call"]("react_agent", carrier=parent_carrier) as task_span: + span = getattr(task_span, "_span", None) + if span is not None and hasattr(span, "set_attribute"): + span.set_attribute("agent.name", agent) + annotate(runtime, input_data={"agent": agent}, output_data={"agent": agent}, tags={"agent": agent}) + child_carrier = carrier_from_task_span(parent_carrier, task_span) + if child_carrier: + return child_carrier + except Exception: + return {} + return {} + + +def resolve_agent_carrier( + *, + state_path: Path, + sid: str, + runtime: dict[str, Any], + parent_carrier: dict[str, str], + agent: str, +) -> dict[str, str]: + """从共享状态里获取或创建持久化的 agent 聚合 carrier。""" + normalized_agent = str(agent or "main").strip() or "main" + existing = st.get_agent_span_carrier(state_path, sid, normalized_agent) + if isinstance(existing, dict) and existing.get("traceparent"): + return dict(existing) + + def _factory() -> dict[str, str]: + return generate_agent_carrier(runtime, parent_carrier=parent_carrier, agent=normalized_agent) + + effective_carrier = st.upsert_agent_span_atomic(state_path, sid, normalized_agent, carrier_factory=_factory) + if not isinstance(effective_carrier, dict) or not effective_carrier.get("traceparent"): + return dict(parent_carrier) + agent_parts = traceparent_parts(effective_carrier) + parent_parts = traceparent_parts(parent_carrier) + debug_write_span( + state_path, + sid, + { + "event": "span_emit", + "span_kind": "agent", + "span_name": "react_agent", + "agent": normalized_agent, + "trace_id": agent_parts.get("trace_id", parent_parts.get("trace_id", "")), + "span_id": agent_parts.get("span_id", ""), + "parent_span_id": parent_parts.get("span_id", ""), + "parent_traceparent": parent_parts.get("traceparent", ""), + }, + sanitizer=sanitize_payload, + ) + return dict(effective_carrier) + + +def resolve_step_carrier( + *, + state_path: Path, + sid: str, + runtime: dict[str, Any], + parent_carrier: dict[str, str], + agent: str, + message_id: str | None, + transcript_path: str, +) -> dict[str, str] | None: + """获取或创建按 message 分组使用的 step carrier。""" + normalized_message_id = str(message_id or "").strip() + if not normalized_message_id: + return None + existing = st.get_step_span_carrier(state_path, sid, agent, normalized_message_id) + if isinstance(existing, dict) and existing.get("traceparent"): + return dict(existing) + + def _factory() -> dict[str, str]: + return generate_step_carrier( + runtime, + parent_carrier=parent_carrier, + agent=agent, + message_id=normalized_message_id, + transcript_path=transcript_path, + ) + + effective_carrier = st.upsert_step_span_atomic( + state_path, + sid, + agent, + normalized_message_id, + carrier_factory=_factory, + transcript_path=transcript_path, + ) + if not isinstance(effective_carrier, dict) or not effective_carrier.get("traceparent"): + return None + step_parts = traceparent_parts(effective_carrier) + parent_parts = traceparent_parts(parent_carrier) + debug_write_span( + state_path, + sid, + { + "event": "span_emit", + "span_kind": "step", + "span_name": "react_step", + "message_id": normalized_message_id, + "agent": agent, + "trace_id": step_parts.get("trace_id", parent_parts.get("trace_id", "")), + "span_id": step_parts.get("span_id", ""), + "parent_span_id": parent_parts.get("span_id", ""), + "parent_traceparent": parent_parts.get("traceparent", ""), + "transcript_path": transcript_path or "", + }, + sanitizer=sanitize_payload, + ) + return dict(effective_carrier) + + +def set_agent_aggregate_on_span(span: Any, aggregate: dict[str, Any] | None) -> None: + """把 agent 的累计指标投影成当前 span 的属性。""" + if span is None or not hasattr(span, "set_attribute") or not isinstance(aggregate, dict): + return + tokens = aggregate.get("tokens") or {} + if isinstance(tokens, dict): + for key in ("input", "output", "cache_read", "cache_creation", "total"): + if key in tokens: + span.set_attribute(f"agent.cumulative_tokens.{key}", int(tokens.get(key) or 0)) + for key in ("cost_usd", "tool_duration_ms", "llm_call_count", "tool_call_count", "step_count", "event_count"): + if key in aggregate: + try: + span.set_attribute(f"agent.cumulative.{key}", int(aggregate.get(key) or 0)) + except (TypeError, ValueError): + span.set_attribute(f"agent.cumulative.{key}", str(aggregate.get(key))) + + +def emit_assistant_span( + runtime: dict[str, Any], + *, + carrier: dict[str, str], + agent: str, + message_id: str | None, + assistant_text: str, + model: str | None = None, + transcript_path: str | None = None, + state_path: Path | None = None, + sid: str | None = None, +) -> None: + """发出一个轻量 assistant span，并按需补上 tool calls。""" + if not assistant_text or not assistant_text.strip(): + return + tool_calls: list[dict[str, Any]] = [] + if message_id and transcript_path: + tool_calls = collect_tool_calls_for_message(transcript_path, message_id) + output_data: dict[str, Any] = {"role": "assistant", "content": assistant_text[:8000]} + if tool_calls: + output_data["tool_calls"] = tool_calls + ctx = runtime["TracerWrapper"].extract_context(carrier) + token = runtime["otel_attach"](ctx) + try: + with runtime["get_tracer"]() as tracer: + with tracer.start_as_current_span(name="assistant_message") as span: + span.set_attribute(runtime["ZhiyanSpanAttributes"].LLM_SPAN_KIND, runtime["ZhiyanllmSpanKindValues"].LLM.value) + span_ids = span_context_ids(span) + parent_parts = traceparent_parts(carrier) + debug_write_span( + state_path, + sid, + { + "event": "span_emit", + "span_kind": "assistant", + "span_name": "assistant_message", + "message_id": str(message_id or ""), + "agent": agent, + "model": str(model or ""), + "tool_call_count": len(tool_calls), + "trace_id": span_ids.get("trace_id", parent_parts.get("trace_id", "")), + "span_id": span_ids.get("span_id", ""), + "parent_span_id": parent_parts.get("span_id", ""), + "parent_traceparent": parent_parts.get("traceparent", ""), + }, + sanitizer=sanitize_payload, + ) + annotate( + runtime, + input_data={"agent": agent, "message_id": message_id or "", "model": model or ""}, + output_data=output_data, + tags={"message_id": str(message_id or ""), "agent": agent}, + ) + finally: + runtime["otel_detach"](token) + + +def emit_tool_span( + runtime: dict[str, Any], + *, + carrier: dict[str, str], + tool_name: str, + active_agent: str, + skills: list[str], + rules: list[str], + duration_ms: int | None = None, + tool_details: dict[str, Any] | None = None, + message_id: str | None = None, + step_grouping: str = "fallback", + state_path: Path | None = None, + sid: str | None = None, +) -> None: + """发出 tool span，并以受控大小挂载 tool payload。""" + def _truncate(value: Any, limit: int = 1200) -> Any: + text = value if isinstance(value, str) else None + if text is None: + return value + if len(text) <= limit: + return text + return text[:limit] + "...(truncated)" + + def _maybe_parse_json_text(value: Any) -> Any: + if not isinstance(value, str): + return value + text = value.strip() + if not text or text[0] not in "{[": + return value + try: + return json.loads(text) + except Exception: + return value + + def _parse_bash_result_content(value: Any) -> dict[str, Any] | None: + if not isinstance(value, str) or "Command:" not in value: + return None + normalized_value = value.replace("\\n", "\n") + markers = [ + ("Command:", "command"), + ("Stdout:", "stdout"), + ("Stderr:", "stderr"), + ("Exit Code:", "exit_code"), + ("Signal:", "signal"), + ] + parsed: dict[str, Any] = {} + for idx, (marker, key) in enumerate(markers): + start = normalized_value.find(marker) + if start < 0: + continue + start += len(marker) + end = len(normalized_value) + for next_marker, _ in markers[idx + 1:]: + pos = normalized_value.find(next_marker, start) + if pos >= 0: + end = min(end, pos) + segment = normalized_value[start:end].strip() + if not segment: + continue + parsed[key] = _truncate(segment, 1200 if key in {"command", "stdout", "stderr"} else 200) + return parsed or None + + tool_input: dict[str, Any] = {"tool": tool_name, "agent": active_agent} + if skills: + tool_input["skill"] = ", ".join(skills) + if rules: + tool_input["rule"] = ", ".join(rules) + if message_id: + tool_input["message_id"] = message_id + details = dict(tool_details or {}) + call_id = str(details.get("call_id") or "").strip() or None + if call_id: + tool_input["call_id"] = call_id + arguments_display_text = details.get("arguments_display_text") + if arguments_display_text: + tool_input["arguments_display_text"] = _truncate(arguments_display_text, 600) + arguments = details.get("arguments") + if arguments: + parsed_arguments = _maybe_parse_json_text(arguments) + tool_input["arguments"] = _truncate(parsed_arguments, 1200) + if isinstance(parsed_arguments, dict): + if parsed_arguments.get("command"): + tool_input["command"] = _truncate(parsed_arguments.get("command"), 1200) + if parsed_arguments.get("description"): + tool_input["description"] = _truncate(parsed_arguments.get("description"), 400) + + tool_output: dict[str, Any] = {"result": "executed"} + result_content = details.get("result_content") + if result_content is not None: + structured_result = _parse_bash_result_content(result_content) + if structured_result: + tool_output["result"] = structured_result + else: + tool_output["result_content"] = _truncate(result_content, 1600) + output_text = details.get("output_text") + if output_text: + tool_output["output_text"] = _truncate(output_text, 1600) + raw_response = details.get("raw_response") + if isinstance(raw_response, dict): + tool_output["raw_response"] = { + "exitCode": raw_response.get("exitCode"), + "signal": raw_response.get("signal"), + "interrupted": raw_response.get("interrupted"), + "sandboxDenied": raw_response.get("sandboxDenied"), + "tool_error_code": raw_response.get("tool_error_code"), + } + + ctx = runtime["TracerWrapper"].extract_context(carrier) + token = runtime["otel_attach"](ctx) + try: + with runtime["get_tracer"]() as tracer: + with tracer.start_as_current_span(name=f"{tool_name}.TOOL") as span: + span_kind_value = runtime["ZhiyanllmSpanKindValues"].TOOL.value + span.set_attribute(zhiyan_attr(runtime, "LLM_SPAN_KIND", "gen_ai.span.kind"), span_kind_value) + span.set_attribute("gen_ai.span.kind", span_kind_value) + span.set_attribute(zhiyan_attr(runtime, "TOOL_NAME", "tool.name"), tool_name) + if isinstance(tool_details, dict): + desc = tool_details.get("description") or tool_details.get("tool_description") + if isinstance(desc, str) and desc.strip(): + span.set_attribute(zhiyan_attr(runtime, "TOOL_DESCRIPTION", "tool.description"), desc.strip()) + tool_params = tool_input.get("arguments") + if tool_params is not None: + if not isinstance(tool_params, str): + tool_params = json.dumps(tool_params, ensure_ascii=False, default=str) + span.set_attribute(zhiyan_attr(runtime, "TOOL_PARAMETERS", "tool.parameters"), tool_params) + if duration_ms is not None: + span.set_attribute("tool.duration_ms", int(duration_ms)) + span_ids = span_context_ids(span) + parent_parts = traceparent_parts(carrier) + debug_write_span( + state_path, + sid, + { + "event": "span_emit", + "span_kind": "tool", + "span_name": f"{tool_name}.TOOL", + "tool": tool_name, + "message_id": str(message_id or ""), + "call_id": call_id or "", + "step_grouping": step_grouping, + "agent": active_agent, + "trace_id": span_ids.get("trace_id", parent_parts.get("trace_id", "")), + "span_id": span_ids.get("span_id", ""), + "parent_span_id": parent_parts.get("span_id", ""), + "parent_traceparent": parent_parts.get("traceparent", ""), + }, + sanitizer=sanitize_payload, + ) + annotate( + runtime, + input_data=tool_input, + output_data=tool_output, + tags={"step_grouping": step_grouping, "message_id": str(message_id or "")}, + ) + finally: + runtime["otel_detach"](token) diff --git a/.claude/skills/agent-observability/scripts/core/cls_sink.py b/.claude/skills/agent-observability/scripts/core/cls_sink.py new file mode 100644 index 0000000..6525f6e --- /dev/null +++ b/.claude/skills/agent-observability/scripts/core/cls_sink.py @@ -0,0 +1,470 @@ +from __future__ import annotations + +import hashlib +import hmac +import json +import os +import struct +import subprocess +import time +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from .agentlens import bootstrap + + + +@dataclass(frozen=True) +class CLSConfig: + enabled: bool + endpoint: str + topic_id: str + secret_id: str + secret_key: str + secret_token: str + service_name: str + timeout_seconds: int + helper_path: Path + sdk_entry_path: Path + + @property + def ready(self) -> bool: + return bool( + self.enabled + and self.endpoint + and self.topic_id + and self.secret_id + and self.secret_key + and self.helper_path.is_file() + and self.sdk_entry_path.is_file() + ) + + +def repo_root() -> Path: + return Path(__file__).resolve().parents[5] + + +def helper_path() -> Path: + return Path(__file__).with_name("cls_uploader.mjs") + + +def sdk_entry_path() -> Path: + return ( + repo_root() + / "cls-codebuddy" + / "tencentcloud-cls-sdk-codebuddy" + / "node_modules" + / "tencentcloud-cls-sdk-js" + / "dist" + / "index.js" + ) + + +def debug_log_path() -> Path: + return Path(__file__).resolve().parents[2] / "logs" / "cls-push-debug.ndjson" + + +def _value( + name: str, + *, + env_local: dict[str, str], + env_file: dict[str, str], + default: str = "", + aliases: tuple[str, ...] = (), +) -> str: + keys = (name, *aliases) + for key in keys: + raw = os.environ.get(key) + if raw is not None and str(raw).strip(): + return str(raw).strip() + for key in keys: + raw = env_local.get(key) + if raw is not None and str(raw).strip(): + return str(raw).strip() + for key in keys: + raw = env_file.get(key) + if raw is not None and str(raw).strip(): + return str(raw).strip() + return default + + +def _enabled_flag(*, env_local: dict[str, str], env_file: dict[str, str]) -> bool: + raw = _value( + "CLS_CODINGAGENT_ENABLED", + env_local=env_local, + env_file=env_file, + default="1", + ) + return str(raw).strip().lower() not in {"0", "false", "off", "no"} + + +def _debug_write(payload: dict[str, Any]) -> None: + try: + path = debug_log_path() + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as fp: + fp.write(json.dumps(payload, ensure_ascii=False) + "\n") + except Exception: + return + + +# ---- CLS PutLogs 的 protobuf 编码 ---- +# CLS 使用了简化版 protobuf： +# LogGroup { repeated Log logs = 1; optional string filename = 2; } +# Log { optional uint32 time = 1; repeated Content contents = 2; } +# Content { optional string key = 1; optional string value = 2; } + +def _encode_varint(value: int) -> bytes: + result = b"" + while value > 0x7F: + result += bytes([(value & 0x7F) | 0x80]) + value >>= 7 + result += bytes([value & 0x7F]) + return result + +def _encode_field(field_number: int, wire_type: int, data: bytes) -> bytes: + tag = (field_number << 3) | wire_type + return _encode_varint(tag) + data + +def _encode_string_field(field_number: int, value: str) -> bytes: + encoded = value.encode("utf-8") + return _encode_field(field_number, 2, _encode_varint(len(encoded)) + encoded) + +def _encode_uint32_field(field_number: int, value: int) -> bytes: + return _encode_field(field_number, 0, _encode_varint(value)) + +def _encode_content(key: str, value: str) -> bytes: + msg = b"" + if key: + msg += _encode_string_field(1, key) + if value: + msg += _encode_string_field(2, value) + return msg + +def _encode_log(record: dict[str, Any], service_name: str) -> bytes: + ts = record.get("ts") + if isinstance(ts, (int, float)): + log_time = int(ts) if ts < 1_000_000_000_000 else int(ts / 1000) + else: + log_time = int(time.time()) + + msg = _encode_uint32_field(1, log_time) + + merged = {"service_name": service_name, **record} + for key, value in merged.items(): + if value is None: + continue + if isinstance(value, (dict, list)): + value = json.dumps(value, ensure_ascii=False) + else: + value = str(value) + msg += _encode_field(2, 2, _encode_varint(len(_encode_content(key, value))) + _encode_content(key, value)) + + return msg + +def _encode_log_group(records: list[dict[str, Any]], service_name: str) -> bytes: + msg = b"" + for record in records: + log_bytes = _encode_log(record, service_name) + msg += _encode_field(1, 2, _encode_varint(len(log_bytes)) + log_bytes) + if service_name: + msg += _encode_string_field(2, service_name) + return msg + +def _encode_log_group_list(records: list[dict[str, Any]], service_name: str) -> bytes: + """编码 LogGroupList protobuf：`message LogGroupList { repeated LogGroup logGroupList = 1; }`。""" + log_group_bytes = _encode_log_group(records, service_name) + return _encode_field(1, 2, _encode_varint(len(log_group_bytes)) + log_group_bytes) + + +# ---- CLS API v3 使用的 TC3-HMAC-SHA256 签名 ---- + +def _hmac_sha256(key: bytes, data: str) -> bytes: + return hmac.new(key, data.encode("utf-8"), hashlib.sha256).digest() + +def _sha256_hex(data: str) -> str: + return hashlib.sha256(data.encode("utf-8")).hexdigest() + +def _put_logs_via_api( + endpoint: str, + topic_id: str, + secret_id: str, + secret_key: str, + log_group_bytes: bytes, + region: str = "ap-guangzhou", + timeout: int = 20, +) -> tuple[bool, str]: + """通过 CLS API v3 发送 UploadLog，并使用 TC3-HMAC-SHA256 签名。 + + `log_group_bytes` 应该是一个 LogGroupList protobuf。 + """ + host = endpoint + service = "cls" + action = "UploadLog" + version = "2020-10-16" + algorithm = "TC3-HMAC-SHA256" + content_type = "application/octet-stream" + + timestamp = int(time.time()) + date_str = time.strftime("%Y-%m-%d", time.gmtime(timestamp)) + + # 请求体就是原始 protobuf 字节串（LogGroupList） + payload = log_group_bytes + hashed_payload = hashlib.sha256(payload).hexdigest() + + # 规范化请求串 + canonical_headers = f"content-type:{content_type}\nhost:{host}\nx-tc-action:{action.lower()}\n" + signed_headers = "content-type;host;x-tc-action" + canonical_request = f"POST\n/\n\n{canonical_headers}\n{signed_headers}\n{hashed_payload}" + + # 待签名字符串 + credential_scope = f"{date_str}/{service}/tc3_request" + hashed_canonical_request = hashlib.sha256(canonical_request.encode("utf-8")).hexdigest() + string_to_sign = f"{algorithm}\n{timestamp}\n{credential_scope}\n{hashed_canonical_request}" + + # 签名结果 + secret_date = _hmac_sha256(("TC3" + secret_key).encode("utf-8"), date_str) + secret_service = _hmac_sha256(secret_date, service) + secret_signing = _hmac_sha256(secret_service, "tc3_request") + signature = hmac.new(secret_signing, string_to_sign.encode("utf-8"), hashlib.sha256).hexdigest() + + authorization = f"{algorithm} Credential={secret_id}/{credential_scope}, SignedHeaders={signed_headers}, Signature={signature}" + + # 构造 HTTP 请求 + url = f"https://{host}" + headers = { + "Authorization": authorization, + "Content-Type": content_type, + "Host": host, + "X-TC-Action": action, + "X-TC-Timestamp": str(timestamp), + "X-TC-Version": version, + "X-TC-Region": region, + "X-CLS-TopicId": topic_id, + "Content-Length": str(len(payload)), + } + + req = urllib.request.Request(url, data=payload, headers=headers, method="POST") + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + body = resp.read().decode("utf-8", errors="replace") + if resp.status == 200: + return True, body + else: + return False, f"HTTP {resp.status}: {body}" + except urllib.error.HTTPError as e: + body = e.read().decode("utf-8", errors="replace")[:500] + return False, f"HTTP {e.code}: {body}" + except Exception as e: + return False, f"{type(e).__name__}: {e}" + + +def _int_value( + name: str, + *, + env_local: dict[str, str], + env_file: dict[str, str], + default: int, + aliases: tuple[str, ...] = (), +) -> int: + raw = _value(name, env_local=env_local, env_file=env_file, default=str(default), aliases=aliases) + try: + parsed = int(raw) + except Exception: + return default + return parsed if parsed > 0 else default + + +def load_config(cwd: str | None = None) -> CLSConfig: + project_root = bootstrap.project_root(cwd) + env_file = bootstrap.read_env_file(project_root / ".env") + env_local = bootstrap.read_env_file(project_root / ".env.local") + return CLSConfig( + enabled=_enabled_flag(env_local=env_local, env_file=env_file), + endpoint=_value( + "CLS_ENDPOINT", + env_local=env_local, + env_file=env_file, + ), + topic_id=_value( + "CLS_TOPIC_ID", + env_local=env_local, + env_file=env_file, + ), + secret_id=_value( + "CLS_SECRET_ID", + env_local=env_local, + env_file=env_file, + aliases=( + "TC_SECRET_ID", + "TENCENTCLOUD_SECRET_ID", + "TENCENTCLOUD_SECRET_ID_438167613", + ), + ), + secret_key=_value( + "CLS_SECRET_KEY", + env_local=env_local, + env_file=env_file, + aliases=( + "TC_SECRET_KEY", + "TENCENTCLOUD_SECRET_KEY", + "TENCENTCLOUD_SECRET_KEY_438167613", + ), + ), + secret_token=_value( + "CLS_SECRET_TOKEN", + env_local=env_local, + env_file=env_file, + aliases=("TC_SECRET_TOKEN", "TC_SESSION_TOKEN", "CLS_SESSION_TOKEN"), + ), + service_name=_value( + "CLS_SERVICE_NAME", + env_local=env_local, + env_file=env_file, + ), + timeout_seconds=_int_value( + "CLS_UPLOAD_TIMEOUT_SECONDS", + env_local=env_local, + env_file=env_file, + default=0, + aliases=("CLS_TIMEOUT_SECONDS",), + ), + helper_path=helper_path(), + sdk_entry_path=sdk_entry_path(), + ) + + +def mirror_record(record: dict[str, Any], *, cwd: str | None = None) -> bool: + config = load_config(cwd) + if not config.ready: + _debug_write( + { + "stage": "config_not_ready", + "endpoint": config.endpoint, + "topic_id": config.topic_id, + "secret_id_present": bool(config.secret_id), + "secret_key_present": bool(config.secret_key), + "secret_token_present": bool(config.secret_token), + "timeout_seconds": config.timeout_seconds, + "helper_exists": config.helper_path.is_file(), + "sdk_exists": config.sdk_entry_path.is_file(), + "event": record.get("event"), + "sid": record.get("sid"), + } + ) + return False + + _debug_write( + { + "stage": "uploader_start", + "endpoint": config.endpoint, + "topic_id": config.topic_id, + "timeout_seconds": config.timeout_seconds, + "event": record.get("event"), + "sid": record.get("sid"), + } + ) + + # 先尝试 Python 原生 API v3 上报（兼容内网 endpoint） + try: + log_group_bytes = _encode_log_group_list([record], config.service_name) + success, detail = _put_logs_via_api( + endpoint=config.endpoint, + topic_id=config.topic_id, + secret_id=config.secret_id, + secret_key=config.secret_key, + log_group_bytes=log_group_bytes, + timeout=config.timeout_seconds, + ) + if success: + _debug_write( + { + "stage": "uploader_ok", + "endpoint": config.endpoint, + "topic_id": config.topic_id, + "method": "python_api_v3", + "event": record.get("event"), + "sid": record.get("sid"), + } + ) + return True + else: + _debug_write( + { + "stage": "uploader_failed", + "method": "python_api_v3", + "error": detail[:2000], + "timeout_seconds": config.timeout_seconds, + "event": record.get("event"), + "sid": record.get("sid"), + } + ) + except Exception as err: + _debug_write( + { + "stage": "uploader_exception", + "method": "python_api_v3", + "error": f"{type(err).__name__}: {err}", + "timeout_seconds": config.timeout_seconds, + "event": record.get("event"), + "sid": record.get("sid"), + } + ) + + # 回退方案：使用 node SDK uploader + payload = { + "endpoint": config.endpoint, + "topicId": config.topic_id, + "secretId": config.secret_id, + "secretKey": config.secret_key, + "secretToken": config.secret_token, + "serviceName": config.service_name, + "records": [record], + } + try: + completed = subprocess.run( + ["node", str(config.helper_path), str(config.sdk_entry_path)], + input=json.dumps(payload, ensure_ascii=False), + text=True, + capture_output=True, + check=False, + timeout=config.timeout_seconds, + ) + if completed.returncode != 0: + _debug_write( + { + "stage": "uploader_failed", + "returncode": completed.returncode, + "stdout": (completed.stdout or "")[:2000], + "stderr": (completed.stderr or "")[:2000], + "timeout_seconds": config.timeout_seconds, + "event": record.get("event"), + "sid": record.get("sid"), + } + ) + return False + except Exception as err: + _debug_write( + { + "stage": "uploader_exception", + "error": f"{type(err).__name__}: {err}", + "timeout_seconds": config.timeout_seconds, + "event": record.get("event"), + "sid": record.get("sid"), + } + ) + return False + + _debug_write( + { + "stage": "uploader_ok", + "endpoint": config.endpoint, + "topic_id": config.topic_id, + "timeout_seconds": config.timeout_seconds, + "event": record.get("event"), + "sid": record.get("sid"), + } + ) + return True diff --git a/.claude/skills/agent-observability/scripts/core/cls_uploader.mjs b/.claude/skills/agent-observability/scripts/core/cls_uploader.mjs new file mode 100644 index 0000000..5c5dccc --- /dev/null +++ b/.claude/skills/agent-observability/scripts/core/cls_uploader.mjs @@ -0,0 +1,96 @@ +import { createRequire } from 'node:module'; + +const MAX_FIELD_LENGTH = 32 * 1024; + +function truncate(value) { + if (value.length <= MAX_FIELD_LENGTH) { + return value; + } + return `${value.slice(0, MAX_FIELD_LENGTH)}...[truncated]`; +} + +function stringifyValue(value) { + if (value === null || value === undefined) { + return ''; + } + if (typeof value === 'string') { + return truncate(value); + } + if (typeof value === 'number' || typeof value === 'boolean') { + return String(value); + } + try { + return truncate(JSON.stringify(value)); + } catch { + return truncate(String(value)); + } +} + +function timestampSeconds(record) { + const raw = Number(record?.ts); + if (!Number.isFinite(raw) || raw <= 0) { + return Math.floor(Date.now() / 1000); + } + return raw > 1_000_000_000_000 ? Math.floor(raw / 1000) : Math.floor(raw); +} + +async function readStdin() { + const chunks = []; + for await (const chunk of process.stdin) { + chunks.push(chunk); + } + return Buffer.concat(chunks).toString('utf8'); +} + +async function main() { + const sdkPath = process.argv[2]; + if (!sdkPath) { + process.exitCode = 2; + process.stderr.write('missing sdk path\n'); + return; + } + + const require = createRequire(import.meta.url); + const { AsyncClient, LogItem, Content, LogGroup, PutLogsRequest } = require(sdkPath); + + const raw = await readStdin(); + const payload = JSON.parse(raw || '{}'); + const records = Array.isArray(payload.records) ? payload.records : []; + if (!payload.endpoint || !payload.topicId || !payload.secretId || !payload.secretKey || records.length === 0) { + process.exitCode = 0; + return; + } + + const client = new AsyncClient({ + endpoint: payload.endpoint, + secretId: payload.secretId, + secretKey: payload.secretKey, + secretToken: payload.secretToken || '', + sourceIp: '127.0.0.1', + retry_times: 3, + }); + + const logGroup = new LogGroup(); + logGroup.setFilename(payload.serviceName || 'agent-observability'); + + for (const record of records) { + const item = new LogItem(); + const merged = { + service_name: payload.serviceName || 'agent-observability', + ...record, + }; + for (const [key, value] of Object.entries(merged)) { + item.pushBack(new Content(key, stringifyValue(value))); + } + item.setTime(timestampSeconds(record)); + logGroup.addLogs(item); + } + + const request = new PutLogsRequest(payload.topicId, logGroup); + await client.PutLogs(request); +} + +main().catch((error) => { + process.exitCode = 1; + process.stderr.write(`${error?.message || String(error)}\n`); +}); diff --git a/.claude/skills/agent-observability/scripts/core/collector.py b/.claude/skills/agent-observability/scripts/core/collector.py new file mode 100644 index 0000000..cfd681b --- /dev/null +++ b/.claude/skills/agent-observability/scripts/core/collector.py @@ -0,0 +1,1008 @@ +"""通用采集层。 + +这一层负责最基础的观测数据采集： +- 维护 Pre/PostToolUse 的配对关系，计算工具耗时 +- 解析 transcript 增量，提取 usage / model / tool 记录 +- 把 skill/rule 命中情况写入 session state +""" +from __future__ import annotations + +import json +import os +import time +from pathlib import Path +from typing import Any + +from . import agent_identity, scanner, state as st + +# 需要从 transcript 中提取的 token 字段 +TOKEN_KEYS = ( + "input_tokens", "output_tokens", "cache_read_input_tokens", + "cache_creation_input_tokens", "total_tokens", +) + +# transcript 记录里可能携带模型名的字段 +MODEL_KEYS = ("model", "requestModelName", "requestModelId") +MESSAGE_ID_KEYS = ("messageId", "responseId", "requestId") + + +def transcript_path(data: dict[str, Any]) -> str: + """从 hook payload 中提取 transcript 路径字段。""" + return str(data.get("transcript_path") or "") + + +def read_stdin_json() -> dict: + """从 stdin 读取 JSON；失败时返回空字典。""" + import sys + try: + raw = sys.stdin.read() + if raw.strip(): + d = json.loads(raw) + if isinstance(d, dict): + return d + return {} + except Exception as e: + return {"_parse_error": str(e)} + + +def record_pre(pending_path: Path, data: dict) -> None: + """记录一条 PreToolUse，供后续 PostToolUse 计算耗时。""" + pending = st.load_pending(pending_path) + sid = data.get("session_id", "?") + tool = data.get("tool_name", "?") + key = f"{sid}::{tool}::{time.time_ns()}" + pending[key] = { + "start": time.time(), + "tool_name": tool, + "session_id": sid, + } + # 只保留最近 20 条 pending 记录 + if len(pending) > 20: + for k in list(pending.keys())[:-20]: + pending.pop(k, None) + st.save_pending(pending_path, pending) + + +def record_post(pending_path: Path, data: dict) -> int | None: + """把 PostToolUse 与之前的 PreToolUse 配对，并返回耗时毫秒数。""" + pending = st.load_pending(pending_path) + tool = data.get("tool_name") + sid = data.get("session_id", "?") + candidates = [(k, v) for k, v in pending.items() + if v.get("session_id") == sid and v.get("tool_name") == tool] + if not candidates: + return None + candidates.sort(key=lambda kv: kv[1]["start"]) + key, item = candidates[-1] + duration_ms = int((time.time() - item["start"]) * 1000) + pending.pop(key, None) + st.save_pending(pending_path, pending) + return duration_ms + + +def parse_transcript_tail( + path: str, + max_lines: int = 50, + last_offset: int = 0, +) -> dict[str, Any]: + """解析 transcript 的增量 JSONL 内容，提取 usage、model 与 tool 元数据。 + + 返回结构： + { + "tokens": {...} or None, # 最新累计 usage + "prev_tokens": {...} or None, # 倒数第二条累计 usage（兼容字段） + "model": str or None, + "offset": int, # 当前 EOF + "tool_records": [ + {"offset": int, "timestamp_ms": int | None, "tool": str, "call_id": str | None, "message_id": str | None, "kind": str}, + ... + ], + "usage_records": [ + {"offset": int, "tokens": {...}, "model": str | None, "message_id": str | None}, + ... + ], + } + + ``usage_records`` 会保留 ``last_offset`` 之后发现的全部 usage 项， + 这样调用方可以回放窗口内每一次模型请求，而不只是最后一个快照。 + """ + summary: dict[str, Any] = { + "tokens": None, + "prev_tokens": None, + "model": None, + "offset": 0, + "tool_records": [], + "usage_records": [], + } + if not path or not os.path.isfile(path): + return summary + + try: + file_size = os.path.getsize(path) + summary["offset"] = file_size + if file_size <= last_offset: + return summary + + with open(path, "rb") as fp: + read_start = max(0, int(last_offset or 0)) + fp.seek(read_start) + tail_bytes = fp.read() + except Exception: + return summary + + last_model_seen: str | None = None + usage_history: list[dict[str, Any]] = [] + pending_tool_records: list[dict[str, Any]] = [] + cursor = max(0, int(last_offset or 0)) + + for raw_line in tail_bytes.splitlines(keepends=True): + cursor += len(raw_line) + if not raw_line.lstrip().startswith(b"{"): + continue + try: + obj = json.loads(raw_line.decode("utf-8", errors="ignore")) + except Exception: + continue + + rec_type = str(obj.get("type") or "").strip().lower() if isinstance(obj, dict) else "" + message_id = _find_message_id(obj) + m = _find_model(obj) + if m: + last_model_seen = m + line_tool_records = _find_tool_records(obj) + if message_id and rec_type not in {"function_call", "function_call_result"}: + for pending_record in pending_tool_records: + pending_record["next_message_id"] = message_id + pending_tool_records = [] + for tool_record in line_tool_records: + tool_record["offset"] = cursor + tool_record["timestamp_ms"] = _find_timestamp_ms(obj) + tool_record["message_id"] = message_id + summary["tool_records"].append(tool_record) + pending_tool_records.append(tool_record) + usage = _find_usage(obj) + if not usage: + continue + + usage_history.append({ + "offset": cursor, + "tokens": usage, + "model": m or last_model_seen, + "message_id": message_id, + }) + if m: + summary["model"] = m + + if usage_history: + summary["usage_records"] = usage_history + summary["tokens"] = usage_history[-1]["tokens"] + if len(usage_history) >= 2: + summary["prev_tokens"] = usage_history[-2]["tokens"] + + if not summary.get("model") and last_model_seen: + summary["model"] = last_model_seen + return summary + + +def record_tool_usage( + state_path: Path, + sid: str, + data: dict, + skills_meta: dict, + rules_meta: dict, + active_agent: str | None = None, + collect_skills: bool = False, +) -> tuple[list[str], list[str]]: + """把一次 tool 调用命中的 skill/rule 写入 session state。 + + Returns: + (used_skills, used_rules): 本次工具调用命中的 skill/rule 名称列表。 + """ + tool = data.get("tool_name") + + def _update(state: dict[str, Any]) -> tuple[list[str], list[str]]: + sess = st.ensure_session(state, sid) + agent = active_agent or sess.get("current_agent") or "main" + + used_skills: set[str] = set() + used_rules: set[str] = set() + + # 1) 可选的 skill 收集 + if collect_skills: + # 直接 use_skill 调用 + direct = scanner.extract_skill_from_tool_call(data) + if direct: + st.bump_skill(sess, direct, via="use_skill", tool=tool, + meta=skills_meta.get(direct), agent=agent) + used_skills.add(direct) + + # 路径推断 + path_skills, _ = scanner.extract_paths_from_tool_call(data) + for s in path_skills: + st.bump_skill(sess, s, via="path-inferred", tool=tool, + meta=skills_meta.get(s), agent=agent) + used_skills.add(s) + + # 子模块命中 + for hit in scanner.extract_submodule_hits(data): + skill_name = hit["skill"] + # unknown 仅代表“命中子模块但无法确定 skill 名称”，不写入实时 skill 字段 + if not skill_name or skill_name == "unknown": + continue + st.bump_skill(sess, skill_name, via="submodule", tool=tool, + meta=skills_meta.get(skill_name), + submodule=hit["submodule"], agent=agent) + used_skills.add(skill_name) + + # Bash skill 脚本 + for s in scanner.extract_bash_skill_scripts(data): + st.bump_skill(sess, s, via="bash-script", tool=tool, + meta=skills_meta.get(s), agent=agent) + used_skills.add(s) + + # 2) 基于路径推断的 rule（始终开启） + _, path_rules = scanner.extract_paths_from_tool_call(data) + for r in path_rules: + st.bump_rule(sess, r, via="path-inferred", tool=tool, meta=rules_meta.get(r), agent=agent) + used_rules.add(r) + + # 3) 当前激活的 rule（始终开启） + for r in scanner.active_rules_for_call(data, rules_meta, active_agent=agent): + st.bump_rule(sess, r, via="active-rule", tool=tool, meta=rules_meta.get(r), agent=agent) + used_rules.add(r) + + st.prune_sessions(state) + return sorted(used_skills), sorted(used_rules) + + return st.update_state_locked(state_path, _update) + + + +def cache_inventory( + state_path: Path, + sid: str, + skills_meta: dict, + rules_meta: dict, +) -> None: + """把完整的 skill/rule inventory 缓存在 session state 中（在 SessionStart 调用）。""" + def _update(state: dict[str, Any]) -> None: + sess = st.ensure_session(state, sid) + sess["_skills_meta"] = skills_meta + sess["_rules_meta"] = rules_meta + sess["_inventory_scanned_at"] = time.time() + + # 为所有已知 skill/rule 预先初始化 count=0 的 usage 记录 + for name, meta in skills_meta.items(): + rec = sess["skills"].setdefault(name, { + "count": 0, "first_ts": None, "last_ts": None, + "tools": [], "via": ["static-scanned"], + "source": meta.get("source"), "version": meta.get("version"), + }) + if not rec.get("source"): + rec["source"] = meta.get("source") + if "static-scanned" not in rec.get("via", []): + rec.setdefault("via", []).append("static-scanned") + + for name, meta in rules_meta.items(): + rec = sess["rules"].setdefault(name, { + "count": 0, "first_ts": None, "last_ts": None, + "tools": [], "via": ["static-scanned"], "source": meta.get("source"), + }) + if "static-scanned" not in rec.get("via", []): + rec.setdefault("via", []).append("static-scanned") + if not rec.get("source"): + rec["source"] = meta.get("source") + + st.prune_sessions(state) + + st.update_state_locked(state_path, _update) + + +def load_cached_inventory(state_path: Path, sid: str) -> tuple[dict, dict]: + """从 state 中读取缓存的 skill/rule inventory，避免在热路径里执行 rglob。""" + state = st.load_state(state_path) + sess = state.get(sid, {}) + if not isinstance(sess, dict): + return {}, {} + return ( + sess.get("_skills_meta") or {}, + sess.get("_rules_meta") or {}, + ) + + +def get_session_usage(state_path: Path, sid: str) -> tuple[dict, dict]: + """返回某个 session 的 skill/rule usage 字典。""" + state = st.load_state(state_path) + sess = state.get(sid, {}) + if not isinstance(sess, dict): + return {}, {} + return sess.get("skills", {}) or {}, sess.get("rules", {}) or {} + +def related_transcript_paths(sid: str, transcript_path: str) -> list[str]: + """收集当前 session 的主 transcript 以及所有 subagent transcript。""" + paths: list[Path] = [] + current = Path(transcript_path).expanduser() if transcript_path else None + if current and current.is_file(): + paths.append(current) + + bundle_dir: Path | None = None + if current: + if current.name == f"{sid}.jsonl": + candidate = current.with_suffix("") + if candidate.is_dir(): + bundle_dir = candidate + else: + for parent in [current.parent, *current.parents]: + if parent.name == sid and parent.is_dir(): + bundle_dir = parent + break + + if bundle_dir is None and current: + candidate = current.parent / sid + if candidate.is_dir(): + bundle_dir = candidate + + if bundle_dir is not None: + main_transcript = bundle_dir.with_suffix(".jsonl") + if main_transcript.is_file(): + paths.append(main_transcript) + subagents_dir = bundle_dir / "subagents" + if subagents_dir.is_dir(): + paths.extend(sorted(path for path in subagents_dir.glob("*.jsonl") if path.is_file())) + + out: list[str] = [] + seen: set[str] = set() + for path in paths: + resolved = str(path.resolve()) + if resolved in seen: + continue + seen.add(resolved) + out.append(resolved) + return out + + +def _transcript_size_signature(paths: list[str]) -> tuple[tuple[str, int], ...]: + """基于路径和文件大小生成 transcript 稳定性签名。""" + signature: list[tuple[str, int]] = [] + for path in paths: + try: + size = Path(path).stat().st_size + except Exception: + size = -1 + signature.append((path, int(size))) + return tuple(signature) + + +def settled_related_transcript_paths( + sid: str, + transcript_path: str, + *, + max_wait_s: float = 2.0, + interval_s: float = 0.4, + stable_rounds: int = 2, +) -> list[str]: + """在 replay 前短暂等待 transcript 文件停止增长。""" + deadline = time.monotonic() + max(0.0, max_wait_s) + previous: tuple[tuple[str, int], ...] | None = None + stable_count = 0 + paths = related_transcript_paths(sid, transcript_path) + + while True: + paths = related_transcript_paths(sid, transcript_path) + current = _transcript_size_signature(paths) + if current == previous: + stable_count += 1 + else: + previous = current + stable_count = 0 + if stable_count >= stable_rounds or time.monotonic() >= deadline: + return paths + sleep_for = min(interval_s, max(0.0, deadline - time.monotonic())) + if sleep_for <= 0: + return paths + time.sleep(sleep_for) + + +def session_id_for_transcript_path(transcript_path: str) -> str | None: + """从 transcript 文件内容中反查 session id。""" + path = Path(transcript_path) + try: + with path.open("r", encoding="utf-8") as fp: + for _ in range(8): + line = fp.readline() + if not line: + break + try: + record = json.loads(line) + except Exception: + continue + session_id = record.get("sessionId") + if isinstance(session_id, str) and session_id.strip(): + return session_id.strip() + except Exception: + return None + return None + + +def resolve_transcript_path_alias(sid: str, transcript_path: str) -> str: + """把别名 transcript 路径解析成当前 session 下的真实文件路径。""" + if not transcript_path: + return "" + current = Path(transcript_path).expanduser() + if current.is_file(): + return str(current.resolve()) + + alias_session_id = current.stem if current.suffix == ".jsonl" else "" + if not alias_session_id or alias_session_id == sid: + return str(current) + + for candidate in related_transcript_paths(sid, transcript_path): + if session_id_for_transcript_path(candidate) == alias_session_id: + return candidate + return str(current) + +def compute_usage_delta(current: dict[str, int], prev: dict[str, int] | None) -> dict[str, int]: + """把一条 usage 记录归一化成当前要发出的 event payload。""" + _ = prev + out = dict(current) + out["total"] = int(out.get("input", 0) or 0) + int(out.get("output", 0) or 0) + return out + + +def normalize_tokens(tokens: dict[str, Any] | None) -> dict[str, int] | None: + """兼容多种 token 字段命名，并统一折叠成一套 schema。""" + if not isinstance(tokens, dict): + return None + + def _int(value: Any) -> int: + try: + return int(value or 0) + except Exception: + return 0 + + input_t = _int(tokens.get("input") if "input" in tokens else tokens.get("input_tokens")) + output_t = _int(tokens.get("output") if "output" in tokens else tokens.get("output_tokens")) + cache_read = _int(tokens.get("cache_read") if "cache_read" in tokens else tokens.get("cache_read_input_tokens")) + cache_creation = _int(tokens.get("cache_creation") if "cache_creation" in tokens else tokens.get("cache_creation_input_tokens")) + total = input_t + output_t + if input_t <= 0 and output_t <= 0 and total <= 0: + return None + out = {"input": input_t, "output": output_t, "cache_read": cache_read, "total": total} + if cache_creation > 0: + out["cache_creation"] = cache_creation + return out + + +def collect_transcript_entries( + state_path: Path, + sid: str, + transcript_path: str, + *, + max_lines: int, +) -> dict[str, Any]: + """读取 transcript 增量，并返回带 source offset 的解析结果。""" + state_data = st.load_state(state_path) + last_offset = st.get_transcript_offset(state_data, sid, transcript_path) + tail = parse_transcript_tail(transcript_path, max_lines=max_lines, last_offset=last_offset) + new_offset = int(tail.get("offset", 0) or 0) + + entries: list[dict[str, Any]] = [] + tool_records: list[dict[str, Any]] = [] + last_cumulative = st.get_last_cumulative_usage(state_data, sid, transcript_path) or None + usage_records = tail.get("usage_records") if isinstance(tail, dict) else None + if not isinstance(usage_records, list): + usage_records = [] + raw_tool_records = tail.get("tool_records") if isinstance(tail, dict) else None + if not isinstance(raw_tool_records, list): + raw_tool_records = [] + + for raw_entry in usage_records: + if not isinstance(raw_entry, dict): + continue + current = normalize_tokens(raw_entry.get("tokens")) + if not current: + continue + delta = compute_usage_delta(current, last_cumulative) + entries.append({ + "offset": int(raw_entry.get("offset", 0) or 0), + "tokens": delta, + "model": str(raw_entry.get("model") or tail.get("model") or "") or None, + "message_id": str(raw_entry.get("message_id") or "").strip() or None, + }) + last_cumulative = current + + for raw_tool_record in raw_tool_records: + if not isinstance(raw_tool_record, dict): + continue + tool_name = str(raw_tool_record.get("tool") or "").strip() + if not tool_name: + continue + tool_records.append({ + "offset": int(raw_tool_record.get("offset", 0) or 0), + "timestamp_ms": int(raw_tool_record.get("timestamp_ms", 0) or 0) or None, + "tool": tool_name, + "call_id": str(raw_tool_record.get("call_id") or "").strip() or None, + "kind": str(raw_tool_record.get("kind") or "").strip() or None, + "message_id": str(raw_tool_record.get("message_id") or "").strip() or None, + "next_message_id": str(raw_tool_record.get("next_message_id") or "").strip() or None, + "arguments": raw_tool_record.get("arguments"), + "arguments_display_text": raw_tool_record.get("arguments_display_text"), + "result_content": raw_tool_record.get("result_content"), + "raw_response": raw_tool_record.get("raw_response"), + "output_text": raw_tool_record.get("output_text"), + }) + + return { + "entries": entries, + "tool_records": tool_records, + "new_offset": new_offset, + "last_cumulative": last_cumulative, + } + + +def find_current_tool_message_id( + state_path: Path, + sid: str, + transcript_path: str, + tool_name: str, + *, + max_lines: int = 80, +) -> str | None: + """找到当前 tool 事件应该归属的 message id。""" + if not transcript_path or not tool_name: + return None + scan = collect_transcript_entries( + state_path=state_path, + sid=sid, + transcript_path=transcript_path, + max_lines=max_lines, + ) + tool_records = scan.get("tool_records") or [] + latest_message_id: str | None = None + latest_offset = -1 + for rec in tool_records: + if not isinstance(rec, dict): + continue + if str(rec.get("tool") or "") != str(tool_name): + continue + message_id = ( + str(rec.get("next_message_id") or "").strip() + or str(rec.get("message_id") or "").strip() + or None + ) + if not message_id: + continue + offset = int(rec.get("offset", 0) or 0) + if offset >= latest_offset: + latest_offset = offset + latest_message_id = message_id + return latest_message_id + + +def tool_details_from_record(rec: dict[str, Any]) -> dict[str, Any]: + """把原始 tool 记录投影成日志里使用的紧凑 detail 结构。""" + details: dict[str, Any] = {} + call_id = str(rec.get("call_id") or "").strip() or None + if call_id: + details["call_id"] = call_id + arguments = rec.get("arguments") + if arguments is not None: + details["arguments"] = arguments + arguments_display_text = rec.get("arguments_display_text") + if arguments_display_text is not None: + details["arguments_display_text"] = arguments_display_text + result_content = rec.get("result_content") + if result_content is not None: + details["result_content"] = result_content + raw_response = rec.get("raw_response") + if isinstance(raw_response, dict) and raw_response: + details["raw_response"] = raw_response + output_text = rec.get("output_text") + if output_text is not None: + details["output_text"] = output_text + next_message_id = str(rec.get("next_message_id") or "").strip() or None + original_message_id = str(rec.get("message_id") or "").strip() or None + if next_message_id and original_message_id: + details["original_message_id"] = original_message_id + if next_message_id: + details["next_message_id"] = next_message_id + if original_message_id and next_message_id and original_message_id != next_message_id: + details["message_id_reassigned"] = True + return details + + +def tool_record_merge_key(rec: dict[str, Any]) -> str: + """为一条 tool record 生成去重合并时使用的稳定 key。""" + call_id = str(rec.get("call_id") or "").strip() + if call_id: + return f"call_id:{call_id}" + offset = int(rec.get("offset", 0) or 0) + tool = str(rec.get("tool") or "").strip() + message_id = str(rec.get("message_id") or "").strip() + timestamp_ms = int(rec.get("timestamp_ms", 0) or 0) + return f"fallback:{tool}:{message_id}:{timestamp_ms}:{offset}" + + +def merge_tool_records(tool_records: list[dict[str, Any]]) -> list[dict[str, Any]]: + """对指向同一次调用的 tool 记录做去重合并。""" + merged: dict[str, dict[str, Any]] = {} + order: list[str] = [] + for rec in tool_records: + if not isinstance(rec, dict): + continue + key = tool_record_merge_key(rec) + if key not in merged: + merged[key] = { + "offset": int(rec.get("offset", 0) or 0), + "timestamp_ms": int(rec.get("timestamp_ms", 0) or 0) or None, + "tool": str(rec.get("tool") or "").strip(), + "call_id": str(rec.get("call_id") or "").strip() or None, + "kind": str(rec.get("kind") or "").strip() or None, + "message_id": str(rec.get("message_id") or "").strip() or None, + "next_message_id": str(rec.get("next_message_id") or "").strip() or None, + "arguments": rec.get("arguments"), + "arguments_display_text": rec.get("arguments_display_text"), + "result_content": rec.get("result_content"), + "raw_response": rec.get("raw_response"), + "output_text": rec.get("output_text"), + } + order.append(key) + continue + + current = merged[key] + current["offset"] = max(int(current.get("offset", 0) or 0), int(rec.get("offset", 0) or 0)) + current_ts = int(current.get("timestamp_ms", 0) or 0) + rec_ts = int(rec.get("timestamp_ms", 0) or 0) + if current_ts <= 0 and rec_ts > 0: + current["timestamp_ms"] = rec_ts + for field in ("tool", "call_id", "kind", "message_id", "next_message_id", "arguments", "arguments_display_text", "result_content", "raw_response", "output_text"): + if current.get(field) is None and rec.get(field) is not None: + current[field] = rec.get(field) + if rec.get("arguments") is not None: + current["arguments"] = rec.get("arguments") + if rec.get("arguments_display_text") is not None: + current["arguments_display_text"] = rec.get("arguments_display_text") + if rec.get("result_content") is not None: + current["result_content"] = rec.get("result_content") + if rec.get("raw_response") is not None: + current["raw_response"] = rec.get("raw_response") + if rec.get("output_text") is not None: + current["output_text"] = rec.get("output_text") + + return [merged[key] for key in order] + + +def tool_context_claim_key(rec: dict[str, Any]) -> str: + """为 tool context 生成 claim 去重键，避免重复消费。""" + call_id = str(rec.get("call_id") or "").strip() + if call_id: + return f"tool_call|{call_id}" + tool = str(rec.get("tool") or "").strip() + message_id = str(rec.get("message_id") or "").strip() + timestamp_ms = int(rec.get("timestamp_ms", 0) or 0) + offset = int(rec.get("offset", 0) or 0) + return f"tool_call|{tool}|{message_id}|{timestamp_ms}|{offset}" + + +def find_current_tool_context( + state_path: Path, + sid: str, + transcript_path: str, + tool_name: str, + *, + call_id: str | None = None, + event_ts: float | None = None, + max_lines: int = 80, + claim: bool = False, +) -> dict[str, Any] | None: + """为当前 tool 调用解析最合适的 transcript 上下文块。""" + if not tool_name: + return None + event_ts_ms = int(float(event_ts) * 1000) if event_ts else None + normalized_call_id = str(call_id or "").strip() or None + + resolved_path = resolve_transcript_path_alias(sid, transcript_path) + candidate_paths: list[str] = [] + if resolved_path: + candidate_paths.append(resolved_path) + for path in related_transcript_paths(sid, transcript_path): + if path not in candidate_paths: + candidate_paths.append(path) + + candidates: list[tuple[tuple[int, int, int], dict[str, Any]]] = [] + for path in candidate_paths: + scan = collect_transcript_entries( + state_path=state_path, + sid=sid, + transcript_path=path, + max_lines=max_lines, + ) + tool_records = merge_tool_records(scan.get("tool_records") or []) + for rec in tool_records: + if not isinstance(rec, dict): + continue + if str(rec.get("tool") or "") != str(tool_name): + continue + rec_call_id = str(rec.get("call_id") or "").strip() or None + if normalized_call_id and rec_call_id != normalized_call_id: + continue + message_id = ( + str(rec.get("next_message_id") or "").strip() + or str(rec.get("message_id") or "").strip() + or None + ) + if not message_id: + continue + offset = int(rec.get("offset", 0) or 0) + timestamp_ms = int(rec.get("timestamp_ms", 0) or 0) + if normalized_call_id: + score = (0, 0, -offset) + elif event_ts_ms and timestamp_ms > 0: + delta = timestamp_ms - event_ts_ms + score = (abs(delta), 0 if delta >= 0 else 1, -offset) + else: + score = (10**12, 1, -offset) + candidates.append(( + score, + { + "message_id": message_id, + "transcript_path": path, + "agent": agent_identity.agent_for_transcript_path(path, "main"), + "pt_id": agent_identity.pt_id_from_transcript_path(path), + "tool_details": tool_details_from_record(rec), + "_claim_key": tool_context_claim_key(rec), + }, + )) + + candidates.sort(key=lambda item: item[0]) + saw_duplicate = False + for _, candidate in candidates: + if not claim: + out = { + "message_id": str(candidate.get("message_id") or ""), + "transcript_path": str(candidate.get("transcript_path") or ""), + "agent": str(candidate.get("agent") or "main"), + "tool_details": dict(candidate.get("tool_details") or {}), + } + pt_id = candidate.get("pt_id") + if pt_id: + out["pt_id"] = pt_id + return out + if st.claim_transcript_event( + state_path, + sid, + str(candidate.get("_claim_key") or ""), + str(candidate.get("transcript_path") or "") or None, + ): + out = { + "message_id": str(candidate.get("message_id") or ""), + "transcript_path": str(candidate.get("transcript_path") or ""), + "agent": str(candidate.get("agent") or "main"), + "tool_details": dict(candidate.get("tool_details") or {}), + } + pt_id = candidate.get("pt_id") + if pt_id: + out["pt_id"] = pt_id + return out + saw_duplicate = True + + if saw_duplicate: + return {"duplicate": True} + return None + + +def extract_tool_call_id(data: dict[str, Any]) -> str | None: + """从事件 payload 或 tool_input 中提取 tool call id。""" + for key in ("call_id", "callId", "tool_call_id", "toolCallId"): + value = data.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + + tool_input = data.get("tool_input") + if isinstance(tool_input, dict): + for key in ("call_id", "callId", "tool_call_id", "toolCallId", "id"): + value = tool_input.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + return None + + +def find_fallback_usage_event( + tool_event: dict[str, Any], + usage_events: list[dict[str, Any]], +) -> dict[str, Any] | None: + """当找不到直接匹配的 tool usage 时，为 AgentLens 选一条兜底 usage。""" + tool_name = str(tool_event.get("tool") or "").strip() + transcript_path = str(tool_event.get("transcript_path") or "").strip() + agent = str(tool_event.get("agent") or "main").strip() or "main" + + strong_matches: list[dict[str, Any]] = [] + weak_matches: list[dict[str, Any]] = [] + for usage_event in usage_events: + if not isinstance(usage_event, dict): + continue + if (str(usage_event.get("agent") or "main").strip() or "main") != agent: + continue + if transcript_path and str(usage_event.get("transcript_path") or "").strip() != transcript_path: + continue + if not str(usage_event.get("message_id") or "").strip(): + continue + + weak_matches.append(usage_event) + event_tool = str(usage_event.get("tool") or "").strip() + if tool_name and event_tool in {tool_name, "model_request"}: + strong_matches.append(usage_event) + + if strong_matches: + return strong_matches[-1] + if weak_matches: + return weak_matches[-1] + return None + + +def build_transcript_event_key(*, kind: str, tool: str, entry: dict[str, Any]) -> str: + """为 transcript 重放出的事件生成幂等键。""" + _ = (kind, tool) + return "|".join(["model_request", str(entry.get("offset", 0) or 0)]) + +def _find_usage(obj: Any, _depth: int = 0) -> dict | None: + """递归查找 JSON 对象里的 token usage。""" + if _depth > 10: + return None + if isinstance(obj, dict): + if "usage" in obj and isinstance(obj["usage"], dict): + u = obj["usage"] + picked = {k: u[k] for k in TOKEN_KEYS if k in u} + if picked: + return picked + for v in obj.values(): + r = _find_usage(v, _depth + 1) + if r: + return r + elif isinstance(obj, list): + for v in obj: + r = _find_usage(v, _depth + 1) + if r: + return r + return None + + +def _find_model(obj: Any, _depth: int = 0) -> str | None: + """递归查找 JSON 对象里的非空模型名。 + + 优先使用 `providerData.model` 或顶层 `model`， + 其次回退到 `requestModelName` / `requestModelId`。 + """ + if _depth > 10: + return None + if isinstance(obj, dict): + for k in MODEL_KEYS: + v = obj.get(k) + if isinstance(v, str) and v.strip(): + return v.strip() + for v in obj.values(): + r = _find_model(v, _depth + 1) + if r: + return r + elif isinstance(obj, list): + for v in obj: + r = _find_model(v, _depth + 1) + if r: + return r + return None + + +def _find_message_id(obj: Any, _depth: int = 0) -> str | None: + """递归查找稳定的 message/request 分组标识。""" + if _depth > 10: + return None + if isinstance(obj, dict): + provider = obj.get("providerData") + if isinstance(provider, dict): + for key in MESSAGE_ID_KEYS: + value = provider.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + extra = obj.get("extra") + if isinstance(extra, dict): + for key in MESSAGE_ID_KEYS: + value = extra.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + for key in MESSAGE_ID_KEYS: + value = obj.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + for v in obj.values(): + r = _find_message_id(v, _depth + 1) + if r: + return r + elif isinstance(obj, list): + for v in obj: + r = _find_message_id(v, _depth + 1) + if r: + return r + return None + + +def _find_timestamp_ms(obj: Any, _depth: int = 0) -> int | None: + """递归查找 transcript 中的毫秒级时间戳。""" + if _depth > 10: + return None + if isinstance(obj, dict): + value = obj.get("timestamp") + if isinstance(value, (int, float)) and value > 0: + return int(value) + provider = obj.get("providerData") + if isinstance(provider, dict): + value = provider.get("timestamp") + if isinstance(value, (int, float)) and value > 0: + return int(value) + for v in obj.values(): + r = _find_timestamp_ms(v, _depth + 1) + if r: + return r + elif isinstance(obj, list): + for v in obj: + r = _find_timestamp_ms(v, _depth + 1) + if r: + return r + return None + + +def _find_tool_records(obj: Any) -> list[dict[str, Any]]: + """从 transcript 记录里提取 tool 调用与 tool 结果片段。""" + out: list[dict[str, Any]] = [] + if not isinstance(obj, dict): + return out + + rec_type = str(obj.get("type") or "").strip().lower() + provider = obj.get("providerData") if isinstance(obj.get("providerData"), dict) else {} + if rec_type in {"function_call", "function_call_result"}: + tool_name = str(obj.get("name") or "").strip() + if tool_name: + record = { + "tool": tool_name, + "call_id": str(obj.get("callId") or "").strip() or None, + "kind": rec_type, + } + if rec_type == "function_call": + arguments = obj.get("arguments") + if isinstance(arguments, str) and arguments.strip(): + record["arguments"] = arguments + arguments_display_text = provider.get("argumentsDisplayText") + if isinstance(arguments_display_text, str) and arguments_display_text.strip(): + record["arguments_display_text"] = arguments_display_text.strip() + if rec_type == "function_call_result": + tool_result = provider.get("toolResult") + if isinstance(tool_result, dict): + content = tool_result.get("content") + if content is not None: + record["result_content"] = content + raw_response = tool_result.get("rawResponse") + if isinstance(raw_response, dict): + record["raw_response"] = raw_response + output = obj.get("output") + if isinstance(output, dict): + output_text = output.get("text") + if output_text is not None: + record["output_text"] = output_text + out.append(record) + + content = obj.get("content") + if isinstance(content, list): + for item in content: + if not isinstance(item, dict): + continue + if str(item.get("type") or "").strip().lower() != "tool_use": + continue + tool_name = str(item.get("name") or "").strip() + if not tool_name: + continue + out.append({ + "tool": tool_name, + "call_id": str(item.get("callId") or "").strip() or None, + "kind": "tool_use", + }) + + return out diff --git a/.claude/skills/agent-observability/scripts/core/devflow.py b/.claude/skills/agent-observability/scripts/core/devflow.py new file mode 100644 index 0000000..ab7ef79 --- /dev/null +++ b/.claude/skills/agent-observability/scripts/core/devflow.py @@ -0,0 +1,234 @@ +"""Devflow 感知层（可选增强，不属于通用采集核心）。 + +这一层只在项目实际跑着 `.codebuddy/runtime` 描述的 multi-agents-devflow 工作流时才生效： +- 判断当前 session 是否属于某个 devflow team（`multi-agents-devflow-{task_slug}`） +- 读取该 team 对应的 `workflow-state.json`，投影成精简 stage 快照 +- 和上一次观测到的快照 diff，只把真正变化的 stage 产出为事件 + +任何解析失败（team 目录不存在 / workflow-state.json 缺失或损坏 / 字段缺失）都必须 +优雅降级为 None / 空列表，绝不能让不跑 devflow 的普通项目因为这一层报错。 +""" +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from . import agent_identity, state as st + + +def task_slug_from_team_dir(team_dir: Path) -> str | None: + """从 devflow team 目录名里剥离出 task_slug（固定前缀 `multi-agents-devflow-`）。""" + name = team_dir.name + prefix = agent_identity.TEAM_PREFIX + if not name.startswith(prefix): + return None + slug = name[len(prefix):].strip() + return slug or None + + +def resolve_devflow_context(cwd: str, sid: str, cached_team_dir: str | None = None) -> dict[str, Any] | None: + """判断当前 session 是否处于某次 devflow 运行中；不是则返回 None。 + + 优先复用 `agent_identity.resolve_team_dir` 做 team 发现（Classic 全部场景，以及 + Portable 在 `topology: team` 宿主——目前是 CodeBuddy——下也走同一套 + `multi-agents-devflow-{task_slug}` 命名，可以直接复用，不用区分 edition）。 + + `topology: spawn` 的宿主（Codex/Claude/Cursor 的 Portable 适配器）不创建 team + 目录，找不到时退化成 `_scan_artifacts_for_active_run` 直接扫 `artifacts/` 目录。 + 这个兜底本身就是尽力而为的启发式，不保证唯一/精确，见该函数的说明。 + + `artifacts_dir` 按 `devflow.defaults.yaml` 的默认值 `{project_root}/artifacts/{task_slug}` + 推算——项目若覆写了 `artifacts.root_dir`，这里暂不感知，读取 workflow-state.json + 找不到文件会安全返回 None，不会误报。 + """ + team_dir = agent_identity.resolve_team_dir(cwd, sid, cached_team_dir) + if team_dir is not None: + task_slug = task_slug_from_team_dir(team_dir) + if task_slug: + artifacts_dir = Path(cwd).expanduser() / "artifacts" / task_slug + workflow_state_path = artifacts_dir / "workflow-state.json" + return { + "task_slug": task_slug, + "team_dir": str(team_dir), + "artifacts_dir": str(artifacts_dir), + "workflow_state_path": str(workflow_state_path), + } + return _scan_artifacts_for_active_run(cwd) + + +def _scan_artifacts_for_active_run(cwd: str) -> dict[str, Any] | None: + """没有 team 目录时的兜底发现：直接扫 `{cwd}/artifacts/*/workflow-state.json`。 + + 用于 `topology: spawn` 的宿主（没有 `.codebuddy/teams/` 这类目录可以反查），以及 + Classic 没有真正暴露 `team_create` 时的降级场景。这是启发式，不是精确匹配： + 多个 task_slug 存在时，优先选"还没跑完"的那个；都跑完或都没跑完时选 + `workflow-state.json` 文件 mtime 最新的一个。"跑完"的判定要兼容两套 schema—— + Classic（v1.3）没有顶层 `status` 字段，看 `last_event == "workflow_completed"`； + Portable（v2.0）看顶层 `status in {"completed", "failed"}`。项目里如果同时有多个 + 真正并发、都还没跑完的 devflow 运行，这个兜底可能选错——已知限制，不在这次范围内解决。 + """ + artifacts_root = Path(cwd).expanduser() / "artifacts" + if not artifacts_root.is_dir(): + return None + candidates: list[tuple[tuple[int, float], str, Path]] = [] + try: + children = list(artifacts_root.iterdir()) + except Exception: + return None + for child in children: + if not child.is_dir(): + continue + state_path = child / "workflow-state.json" + state = read_workflow_state(str(state_path)) + if not isinstance(state, dict) or not isinstance(state.get("stages"), dict): + continue + try: + mtime = state_path.stat().st_mtime + except Exception: + mtime = 0.0 + finished = ( + str(state.get("last_event") or "") == "workflow_completed" + or str(state.get("status") or "") in {"completed", "failed"} + ) + not_finished = 0 if finished else 1 + candidates.append(((not_finished, mtime), child.name, state_path)) + if not candidates: + return None + candidates.sort(key=lambda item: item[0], reverse=True) + _, task_slug, state_path = candidates[0] + return { + "task_slug": task_slug, + "team_dir": None, + "artifacts_dir": str(state_path.parent), + "workflow_state_path": str(state_path), + } + + +def read_workflow_state(path: str) -> dict[str, Any] | None: + """安全读取 workflow-state.json；文件不存在或解析失败都返回 None。""" + try: + p = Path(path) + if not p.is_file(): + return None + data = json.loads(p.read_text("utf-8")) + return data if isinstance(data, dict) else None + except Exception: + return None + + +def stage_snapshot(workflow_state: dict[str, Any] | None) -> dict[str, Any]: + """把 workflow-state.json 投影成精简快照，供比对和事件输出使用。 + + 兼容两套 schema：Classic（v1.3，字段名 `executor`，有 `review_result`）和 + Portable（v2.0，字段名 `executor_role`，没有 `review_result`，但顶层多了 + `execution_mode`/`host_adapter`/`run_id`/`team_name` 这些 Classic 没有的上下文）。 + 按 `version` 字段区分，取不到就都尝试取一遍，不强制要求调用方先判断是哪个 edition。 + """ + if not isinstance(workflow_state, dict): + return {} + stages_raw = workflow_state.get("stages") + stages: dict[str, Any] = {} + if isinstance(stages_raw, dict): + for name, info in stages_raw.items(): + if not isinstance(info, dict): + continue + stages[name] = { + "status": info.get("status"), + "executor": info.get("executor") or info.get("executor_role"), + "retry_count": info.get("retry_count", 0), + "review_result": info.get("review_result"), + } + return { + "current_stage": workflow_state.get("current_stage"), + "size_class": workflow_state.get("size_class"), + "run_mode": workflow_state.get("run_mode"), + "schema_version": workflow_state.get("version"), + "execution_mode": workflow_state.get("execution_mode"), + "host_adapter": workflow_state.get("host_adapter"), + "run_id": workflow_state.get("run_id"), + "stages": stages, + } + + +def diff_stage_changes(prev: dict[str, Any] | None, curr: dict[str, Any]) -> list[dict[str, Any]]: + """比较两次 stage 快照，只返回 status/retry_count/review_result 真正变化的阶段。 + + 首次观测（`prev` 为 None，即这个 session 第一次检测到 devflow）不产出任何变更—— + 避免刚接入 observability 时，把一个已经跑了大半的 devflow 运行的全部历史阶段 + 当成"新事件"一次性炸出来。之后每次变化都会被正常捕获。 + """ + if not isinstance(curr, dict) or prev is None: + return [] + curr_stages = curr.get("stages") or {} + prev_stages = prev.get("stages") if isinstance(prev, dict) else {} + if not isinstance(prev_stages, dict): + prev_stages = {} + changes: list[dict[str, Any]] = [] + for name, info in curr_stages.items(): + if not isinstance(info, dict): + continue + before = prev_stages.get(name) + before = before if isinstance(before, dict) else {} + fields = ("status", "retry_count", "review_result") + if any(before.get(f) != info.get(f) for f in fields): + changes.append({ + "stage": name, + "status": info.get("status"), + "executor": info.get("executor"), + "retry_count": info.get("retry_count", 0), + "review_result": info.get("review_result"), + }) + return changes + + +def resolve_and_diff(state_path: Path, sid: str, cwd: str) -> dict[str, Any] | None: + """解析当前 session 归属的 devflow 上下文，返回上下文 + 本次观测到的 stage 变更。 + + **不缓存"当前归属哪个 task_slug"这个结论本身**——同一个 sid 在其生命周期内可能 + 先后归属不同的 devflow 运行。这不是理论场景：当运行时没有暴露真正的 + `team_create`/`send_message`（因此没有独立的 team 成员 sid），devflow 会退化成 + 在同一个长生命周期 session 里，先后跑好几次 `/start-devflow`；每次都必须重新判定 + "现在最合适的 task_slug 是哪个"，而不能沿用第一次探测到的那个——沿用旧结论会把 + 第二次运行的所有事件都错误地归到第一次的 task_slug 下。 + + 重新判定的代价很低：`resolve_devflow_context` 对团队目录的探测会用上一次找到的 + `team_dir` 做快速校验（只在真正失效时才重新扫描 `.codebuddy/teams/`）； + `artifacts/` 兜底扫描也只是一次浅层 `iterdir()`，不是全量遍历。 + + 真正跨调用持久化的只有**按 task_slug 分别保存的 stage 快照历史**——切换到另一个 + task_slug 不会污染对方的 diff 基线，也不会因为切回旧 task_slug 而把它已经观测过的 + 历史重新当成"首次观测"。 + + 返回 None 表示这次调用没有探测到任何 devflow 上下文；否则返回 + `{task_slug, artifacts_dir, workflow_state_path, current_stage, size_class, changes}`。 + """ + + def _update(state: dict[str, Any]) -> dict[str, Any] | None: + dv = st.get_devflow_state(state, sid) + cached_team_dir = (dv.get("context") or {}).get("team_dir") if isinstance(dv.get("context"), dict) else None + context = resolve_devflow_context(cwd, sid, cached_team_dir) + if not isinstance(context, dict): + st.set_devflow_state(state, sid, {"context": None}) + return None + + task_slug = str(context.get("task_slug") or "") + workflow_state = read_workflow_state(context.get("workflow_state_path", "")) + curr = stage_snapshot(workflow_state) + snapshots = dv.get("snapshots") + if not isinstance(snapshots, dict): + snapshots = {} + prev = snapshots.get(task_slug) + changes = diff_stage_changes(prev if isinstance(prev, dict) else None, curr) + snapshots[task_slug] = curr + st.set_devflow_state(state, sid, {"context": context, "snapshots": snapshots}) + + result = dict(context) + result["current_stage"] = curr.get("current_stage") + result["size_class"] = curr.get("size_class") + result["schema_version"] = curr.get("schema_version") + result["execution_mode"] = curr.get("execution_mode") + result["changes"] = changes + return result + + return st.update_state_locked(state_path, _update) diff --git a/.claude/skills/agent-observability/scripts/core/emitter.py b/.claude/skills/agent-observability/scripts/core/emitter.py new file mode 100644 index 0000000..a41dfc0 --- /dev/null +++ b/.claude/skills/agent-observability/scripts/core/emitter.py @@ -0,0 +1,578 @@ +"""ndjson 发射层。 + +只输出当前真正需要的核心字段： +- common: data_source +- tool: event, sid, ts, agent, tool, ms, skill, rule, transcript_path +- usage: event, sid, ts, agent, tool, tokens, model, cost_usd, transcript_path +- start: event, sid, ts, agent +- stop: event, sid, ts, agent, tokens, model, cost_usd, cost_session_usd, transcript_path +- error: event, sid, ts, phase, error + +历史 x_* 扩展字段、stop skill/rule 汇总、schema 版本号等都不再生成。 + +`cost_usd` 的价格表 = 内置 `config/pricing.json` 与用户覆盖文件（默认 +`/.codebuddy/agent-observability/pricing.overrides.json`，可用 +`AOBS_PRICING_OVERRIDES_PATH` 改路径）的字段级合并结果，详见 `load_prices()`。 +""" +from __future__ import annotations + +import json +import os +import time +from functools import lru_cache +from pathlib import Path +from typing import Any + +from . import cls_sink +from . import state as st + +DEFAULT_PRICES_PATH = Path(__file__).resolve().parents[2] / "config" / "pricing.json" +DEFAULT_DATA_SOURCE = "codebuddy-cli" + + +def _default_overrides_path() -> Path | None: + """用户定价覆盖文件的默认位置：`/.codebuddy/agent-observability/pricing.overrides.json`。 + + 刻意放在 `skills/` 树**外**：`scripts/build-classic-hosts.py` 会整棵同步 + `.codebuddy/skills`，放树内会 (a) 每次改动都产生 `--check` drift，(b) 把用户的 + 自定义价格复制进 `.claude`/`.cursor` 宿主包；放树外则既不参与同步，也不会被 + `config/pricing.json` 的后续更新冲掉。 + """ + try: + project_root = Path(__file__).resolve().parents[5] + except IndexError: + return None + return project_root / ".codebuddy" / "agent-observability" / "pricing.overrides.json" + + +# None 表示"拿不到项目根"，此时等同于"没有覆盖文件"（纯内置价格表）。 +DEFAULT_OVERRIDES_PATH: Path | None = _default_overrides_path() + + +def get_log_path(base_dir: Path) -> Path: + """返回 metrics 日志路径，并确保目录已存在。""" + log_dir = base_dir / "logs" + log_dir.mkdir(parents=True, exist_ok=True) + return log_dir / "metrics.ndjson" + + +def _first_record_ts(log_path: Path, sid: str) -> float | None: + """从 metrics.ndjson 中找到某个 session 的最早 ts。 + + state 只保留最近几个 session，长流程或历史 session 可能被 prune 后重建， + 因此 `started_at` 不能作为唯一权威。日志里的首条 ts 更接近 + session/workflow 的真实起点。 + """ + if not sid or not log_path.exists(): + return None + first_ts: float | None = None + try: + with log_path.open("r", encoding="utf-8") as fp: + for line in fp: + if not line.strip().startswith("{"): + continue + try: + rec = json.loads(line) + except Exception: + continue + if rec.get("sid") != sid: + continue + try: + ts = float(rec.get("ts")) + except Exception: + continue + if first_ts is None or ts < first_ts: + first_ts = ts + except Exception: + return None + return first_ts + + +def _inject_session_duration(record: dict[str, Any], log_path: Path) -> None: + """从 state 文件读取 session started_at，计算 session_duration_sec 注入 record。 + 同时扫描 metrics.ndjson 累计 session 级别 token 总量。""" + sid = record.get("sid") + if not sid or "session_duration_sec" in record: + return + try: + now = float(record.get("ts") or time.time()) + started_candidates: list[float] = [] + state_path = log_path.parent / ".state.json" + state = st.load_state(state_path) + sess = state.get(sid) + if isinstance(sess, dict): + try: + state_started_at = float(sess.get("started_at") or 0) + except Exception: + state_started_at = 0.0 + if state_started_at > 0: + started_candidates.append(state_started_at) + log_started_at = _first_record_ts(log_path, str(sid)) + if log_started_at is not None and log_started_at > 0: + started_candidates.append(log_started_at) + if not started_candidates: + started_candidates.append(now) + started_at = min(started_candidates) + record["session_duration_sec"] = round(max(0.0, now - started_at), 3) + except Exception: + return + + # 累计 session 级别 token（扫描 metrics.ndjson 尾部） + try: + total_input = 0 + total_output = 0 + total_cache = 0 + if log_path.exists(): + with log_path.open("r", encoding="utf-8") as fp: + for line in fp: + if not line.strip().startswith("{"): + continue + try: + rec = json.loads(line) + except Exception: + continue + if rec.get("sid") != sid or rec.get("event") != "usage": + continue + tokens = rec.get("tokens") + if not isinstance(tokens, dict): + continue + total_input += int(tokens.get("input") or 0) + total_output += int(tokens.get("output") or 0) + total_cache += int(tokens.get("cache_read") or 0) + record["session_total_tokens"] = total_input + total_output + record["session_input_tokens"] = total_input + record["session_output_tokens"] = total_output + record["session_cache_tokens"] = total_cache + except Exception: + return + + +def emit(log_path: Path, record: dict[str, Any]) -> None: + """向 ndjson 日志追加一条记录。""" + record.setdefault("data_source", DEFAULT_DATA_SOURCE) + record.setdefault("ts", time.time()) + _inject_session_duration(record, log_path) + with log_path.open("a", encoding="utf-8") as fp: + fp.write(json.dumps(record, ensure_ascii=False) + "\n") + try: + cls_sink.mirror_record(record) + except Exception: + return + + +def build_tool_event( + sid: str, + tool: str | None, + duration_ms: int | None, + active_agent: str | None = None, + transcript_path: str | None = None, + turn_id: str | None = None, + task_slug: str | None = None, + stage: str | None = None, + pt_id: str | None = None, +) -> dict[str, Any]: + """构造 tool 事件记录，只包含调用信息与耗时。 + + `task_slug` / `stage` / `pt_id` 只有在检测到 devflow 运行时才会出现（见 + `core/devflow.py`），非 devflow 项目不受影响。 + """ + record = { + "event": "tool", + "sid": sid, + "agent": active_agent or "main", + "tool": tool, + "ms": duration_ms, + } + if transcript_path: + record["transcript_path"] = transcript_path + if turn_id: + record["turn_id"] = turn_id + if task_slug: + record["task_slug"] = task_slug + if stage: + record["stage"] = stage + if pt_id: + record["pt_id"] = pt_id + return record + + +def build_usage_event( + sid: str, + tool: str | None, + tokens: dict[str, Any], + active_agent: str | None = None, + model: str | None = None, + cost_usd: float | None = None, + transcript_path: str | None = None, + source_offset: int | None = None, + turn_id: str | None = None, + message_id: str | None = None, + task_slug: str | None = None, + stage: str | None = None, +) -> dict[str, Any]: + """构造 LLM usage 事件记录。""" + record: dict[str, Any] = { + "event": "usage", + "sid": sid, + "agent": active_agent or "main", + "tool": tool, + "tokens": tokens, + } + if model: + record["model"] = model + if cost_usd is not None: + record["cost_usd"] = cost_usd + if transcript_path: + record["transcript_path"] = transcript_path + if source_offset is not None: + record["source_offset"] = source_offset + if turn_id: + record["turn_id"] = turn_id + if message_id: + record["message_id"] = message_id + if task_slug: + record["task_slug"] = task_slug + if stage: + record["stage"] = stage + return record + + +def build_start_event( + sid: str, + agent: str | None = None, +) -> dict[str, Any]: + """构造最小化的 start 事件记录。 + + 初始化阶段没有明确角色时，统一记为 ``main``。 + """ + return { + "event": "start", + "sid": sid, + "agent": agent or "main", + } + + +def build_prompt_submit_event( + sid: str, + turn_id: str, + agent: str | None = None, + prompt_len: int | None = None, +) -> dict[str, Any]: + """构造 `user_prompt_submit` 事件，作为 turn 边界标记。 + + 这条记录会落到 metrics.ndjson，方便下游把 turn 和 AgentLens trace 对上。 + """ + record: dict[str, Any] = { + "event": "user_prompt_submit", + "sid": sid, + "agent": agent or "main", + "turn_id": turn_id, + } + if prompt_len is not None: + record["prompt_len"] = int(prompt_len) + return record + + +def build_stop_event( + sid: str, + tokens: dict | None = None, + model: str | None = None, + cost_usd: float | None = None, + cost_session_usd: float | None = None, + active_agent: str | None = None, + transcript_path: str | None = None, + source_offset: int | None = None, + turn_id: str | None = None, + message_id: str | None = None, + task_slug: str | None = None, + stage: str | None = None, +) -> dict[str, Any]: + """构造最小化的 stop 事件记录。""" + record: dict[str, Any] = { + "event": "stop", + "sid": sid, + "agent": active_agent or "main", + } + if tokens: + record["tokens"] = tokens + if model: + record["model"] = model + if cost_usd is not None: + record["cost_usd"] = cost_usd + if cost_session_usd is not None: + record["cost_session_usd"] = cost_session_usd + if transcript_path: + record["transcript_path"] = transcript_path + if source_offset is not None: + record["source_offset"] = source_offset + if turn_id: + record["turn_id"] = turn_id + if message_id: + record["message_id"] = message_id + if task_slug: + record["task_slug"] = task_slug + if stage: + record["stage"] = stage + return record + + +def build_error_event( + phase: str, + error: str, + sid: str | None = None, +) -> dict[str, Any]: + """构造最小化的 error 事件记录。""" + return { + "event": "error", + "sid": sid or "", + "phase": phase, + "error": error, + } + + +def build_stage_transition_event( + sid: str, + task_slug: str, + stage: str, + status: str | None = None, + executor: str | None = None, + retry_count: int = 0, + review_result: str | None = None, +) -> dict[str, Any]: + """构造 devflow stage 变更事件（只在检测到 devflow 运行、且 stage 真的变化时发出）。 + + 对应 `workflow-state.json` 里某个 stage 的 `status`/`retry_count`/`review_result` + 发生变化——retry_count 上升或 review_result="failed" 是 devflow 里最值得关注的 + 信号（返工/打回循环），比 hook 自身的 `error` 事件更贴近业务语义。 + """ + record: dict[str, Any] = { + "event": "stage_transition", + "sid": sid, + "task_slug": task_slug, + "stage": stage, + } + if status: + record["status"] = status + if executor: + record["executor"] = executor + if retry_count: + record["retry_count"] = int(retry_count) + if review_result: + record["review_result"] = review_result + return record + + +def resolve_overrides_path() -> Path | None: + """返回当前生效的覆盖文件路径：`AOBS_PRICING_OVERRIDES_PATH` 优先，否则默认路径。 + + 返回 `None` 表示"不加载任何覆盖"（纯内置价格表）。支持 `~` 展开。 + """ + raw = os.environ.get("AOBS_PRICING_OVERRIDES_PATH", "").strip() + if raw: + return Path(raw).expanduser() + return DEFAULT_OVERRIDES_PATH + + +def _coerce_price_table(data: Any) -> dict[str, dict[str, float]]: + """把任意 JSON 结果规整成 `{model: {field: float}}`，坏数据按字段/按模型跳过。 + + 与内置表加载的差别只有一处：内置表遇到非数字会整体抛错降级，这里逐字段跳过—— + 用户的补丁文件里写错一个字段不应该把其余正确的覆盖一起丢掉，更不应该让 hook 挂掉。 + """ + if not isinstance(data, dict): + return {} + table: dict[str, dict[str, float]] = {} + for raw_key, raw_value in data.items(): + if not isinstance(raw_value, dict): + continue + key = str(raw_key).strip().lower() + if not key: + continue + row: dict[str, float] = {} + for price_key, price_value in raw_value.items(): + # bool 是 int 的子类，True 会被 float() 变成 1.0，这里按"不是价格"处理。 + if price_value is None or isinstance(price_value, bool): + continue + try: + row[str(price_key).strip().lower()] = float(price_value) + except (TypeError, ValueError): + continue + # 全是坏字段的模型不落表：否则它会被 `lookup_price` 命中成 {}， + # 既算不出成本又不算"未定价"，看板上会变成一个查不到原因的空洞。 + if row: + table[key] = row + return table + + +def load_price_overrides(path: Path | None = None) -> dict[str, dict[str, float]]: + """加载用户定价覆盖文件；任何异常都静默降级为空覆盖（纯内置价格表）。 + + 静默是刻意的：这个文件是给人手写的常驻配置，hook 每次都是新进程， + 一旦因格式问题抛异常/打印，会让整条 hook 链路变得不可用。 + """ + try: + target = path if path is not None else resolve_overrides_path() + if not target: + return {} + with Path(target).expanduser().open("r", encoding="utf-8") as fp: + data = json.load(fp) + except Exception: + return {} + return _coerce_price_table(data) + + +def merge_prices( + base: dict[str, dict[str, float]] | None, + overrides: dict[str, dict[str, float]] | None, +) -> dict[str, dict[str, float]]: + """字段级合并价格表：覆盖里写出的字段替换内置值，没写的字段沿用内置值。 + + 可以新增内置表里不存在的模型；**不支持删除**内置模型或字段（合并只能加不能减）。 + """ + merged: dict[str, dict[str, float]] = {} + for model, row in (base or {}).items(): + if isinstance(row, dict): + merged[model] = dict(row) + for model, patch in (overrides or {}).items(): + if not isinstance(patch, dict): + continue + merged.setdefault(model, {}).update(patch) + return merged + + +def _load_builtin_prices() -> dict[str, dict[str, float]]: + """从 `config/pricing.json` 中一次性加载内置模型价格配置（不含用户覆盖）。""" + path = os.environ.get("AOBS_PRICES_PATH", "").strip() + prices_path = Path(path).expanduser() if path else DEFAULT_PRICES_PATH + try: + with prices_path.open("r", encoding="utf-8") as fp: + data = json.load(fp) + except Exception: + return {} + if not isinstance(data, dict): + return {} + return { + str(key).lower(): {price_key: float(price_value) for price_key, price_value in value.items()} + for key, value in data.items() + if isinstance(value, dict) + } + + +@lru_cache(maxsize=1) +def load_prices() -> dict[str, dict[str, float]]: + """内置价格表 + 用户覆盖文件的合并结果（字段级）。 + + 结果带 `lru_cache`：hook 是短命进程，一次会话里只该读一次盘；长驻进程或单测里 + 改了覆盖文件/环境变量后，调 `clear_price_cache()` 让下一次调用重新加载。 + """ + return merge_prices(_load_builtin_prices(), load_price_overrides()) + + +def clear_price_cache() -> None: + """丢弃 `load_prices()` 的缓存，让下一次调用重读磁盘与环境变量。""" + load_prices.cache_clear() + + +def lookup_price(model: str | None) -> dict[str, float] | None: + """为一个具体模型名找到最匹配的价格配置项。""" + if not model or not isinstance(model, str): + return None + normalized = model.lower().strip() + if not normalized: + return None + prices = load_prices() + if normalized in prices: + return prices[normalized] + best_key: str | None = None + for key in prices: + if key in normalized and (best_key is None or len(key) > len(best_key)): + best_key = key + if best_key is None: + return None + return prices[best_key] + + +def is_unpriced(model: str | None) -> bool: + """模型是否完全没命中价格表（含模糊匹配）。 + + "未定价"的唯一口径就是 `lookup_price(...) is None`（D4）：模糊（最长子串）命中的 + 模型能算出成本，就不该出现在看板的"建议补价"提示里。 + """ + return lookup_price(model) is None + + +def estimate_cost(tokens: dict | None, model: str | None) -> dict[str, Any] | None: + """根据归一化后的 token 总量估算单条 usage 事件的成本。""" + if not isinstance(tokens, dict): + return None + rates = lookup_price(model) + if not rates: + return None + + def _int(key: str) -> int: + try: + return int(tokens.get(key) or 0) + except Exception: + return 0 + + output = _int("output") + cache_read = _int("cache_read") + cache_write = _int("cache_creation") + raw_input = _int("input") + fresh_input = raw_input - cache_read - cache_write if raw_input and raw_input >= (cache_read + cache_write) else raw_input + cost = ( + fresh_input * rates.get("input", 0.0) + + cache_read * rates.get("cache_read", 0.0) + + cache_write * rates.get("cache_write", 0.0) + + output * rates.get("output", 0.0) + ) / 1_000_000.0 + + matched_key = None + normalized = (model or "").lower() + for key in load_prices(): + if key in normalized and (matched_key is None or len(key) > len(matched_key)): + matched_key = key + if matched_key is None and normalized in load_prices(): + matched_key = normalized + + return { + "usd": round(cost, 6), + "model_matched": matched_key, + "rates": rates, + } + + +def cost_of(tokens: dict | None, model: str | None) -> float | None: + """只返回美元成本值的便捷封装。""" + if not tokens or not model: + return None + info = estimate_cost(tokens, model) + if not info: + return None + return info.get("usd") + + +def sum_session_cost(log_path: Path, sid: str) -> float | None: + """尽力根据已发出的事件反算整个 session 的总成本。""" + if not log_path.exists() or not sid: + return None + total = 0.0 + seen = False + try: + with log_path.open("r", encoding="utf-8") as fp: + for line in fp: + if not line.strip().startswith("{"): + continue + try: + rec = json.loads(line) + except Exception: + continue + if rec.get("sid") != sid or rec.get("event") not in {"usage", "tool"}: + continue + cost = rec.get("cost_usd") + if isinstance(cost, (int, float)): + total += float(cost) + seen = True + except Exception: + return None + return round(total, 6) if seen else None diff --git a/.claude/skills/agent-observability/scripts/core/runtime.py b/.claude/skills/agent-observability/scripts/core/runtime.py new file mode 100644 index 0000000..9c647bc --- /dev/null +++ b/.claude/skills/agent-observability/scripts/core/runtime.py @@ -0,0 +1,598 @@ +from __future__ import annotations +"""Hook 主编排入口。 + +这一层负责把外部 hook phase 分发到内部能力： +- session-start / user-prompt-submit / pre / post / stop +- 组装统一运行时路径 +- 串联 collector / emitter / agentlens，并承担 transcript 运行时编排 +""" + +import os +import re +import time +import traceback +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from . import agent_identity, agentlens, collector, devflow, emitter, scanner, state as st + + +@dataclass(frozen=True) +class RuntimePaths: + """运行时常用路径集合，避免每个分支重复计算。""" + base_dir: Path + state_path: Path + pending_path: Path + log_path: Path + + +def base_dir() -> Path: + """返回 hook 运行时使用的根目录。""" + # 运行时产物统一写到 skill 根目录下的 logs/，保持与旧路径兼容。 + return Path(__file__).resolve().parents[2] + + +def build_runtime_paths() -> RuntimePaths: + """构造本次 hook 运行要用到的路径集合。""" + base = base_dir() + return RuntimePaths( + base_dir=base, + state_path=st.get_state_path(base), + pending_path=st.get_pending_path(base), + log_path=emitter.get_log_path(base), + ) + + +def session_id_of(data: dict[str, Any]) -> str: + """兼容不同字段名，提取 session id。""" + return str(data.get("session_id") or data.get("sid") or "") + + +def cwd(data: dict[str, Any]) -> str: + """解析当前项目工作目录。""" + return str(data.get("cwd") or os.environ.get("CODEBUDDY_PROJECT_DIR") or os.getcwd()) + + +def normalize_phase(raw: str | None) -> str: + """把多种 phase 写法折叠成统一内部枚举。""" + phase = (raw or "").strip().lower() + return { + "sessionstart": "session-start", + "session_start": "session-start", + "session-start": "session-start", + "pretooluse": "pre", + "pre": "pre", + "posttooluse": "post", + "post": "post", + "stop": "stop", + "userpromptsubmit": "user-prompt-submit", + "user_prompt_submit": "user-prompt-submit", + "user-prompt-submit": "user-prompt-submit", + "prompt": "user-prompt-submit", + }.get(phase, phase) + + +def emit_error(phase: str, sid: str, err: Exception) -> None: + """把运行时异常写成 error 事件，而不是直接中断 hook。""" + rec = emitter.build_error_event(phase=phase, error=f"{type(err).__name__}: {err}", sid=sid) + rec["x_traceback"] = traceback.format_exc(limit=5)[-1200:] + emitter.emit(build_runtime_paths().log_path, rec) + + +def derive_turn_id(data: dict[str, Any]) -> str: + """优先复用外部已有 id，否则生成一个本地 turn id。""" + for key in ("turn_id", "prompt_id", "request_id", "message_id"): + raw = data.get(key) + if isinstance(raw, str) and raw.strip(): + return raw.strip() + if isinstance(raw, (int, float)) and raw: + return str(raw) + return f"turn-{int(time.time() * 1000)}" + + +def prompt_text_length(data: dict[str, Any]) -> int | None: + """统计用户 prompt 的文本长度，用于 turn 边界指标。""" + for key in ("prompt", "user_prompt", "message", "content", "text"): + val = data.get(key) + if isinstance(val, str): + return len(val) + if isinstance(val, list): + total = 0 + for item in val: + if isinstance(item, dict): + txt = item.get("text") + if isinstance(txt, str): + total += len(txt) + elif isinstance(item, str): + total += len(item) + if total: + return total + return None + + +def extract_prompt_text(data: dict[str, Any]) -> str: + """从 hook payload 中提取可读的 prompt 文本。""" + for key in ("prompt", "user_prompt", "message", "text"): + val = data.get(key) + if isinstance(val, str) and val.strip(): + return val.strip() + if isinstance(val, list): + parts = [] + for item in val: + if isinstance(item, dict): + txt = item.get("text") + if isinstance(txt, str): + parts.append(txt) + elif isinstance(item, str): + parts.append(item) + if parts: + return "\n".join(parts).strip() + return "" + + +def derive_business_scenario(prompt_text: str) -> str | None: + """从 prompt 文本中派生一个简短业务场景标识。""" + if not prompt_text: + return None + + m = re.search(r"标题[：:]\s*(.+?)(?:\s*描述[：:]|\s*$)", prompt_text, re.DOTALL) + if m: + title = m.group(1).strip()[:60] + else: + title = prompt_text[:50].strip() + + sanitized = re.sub(r"[^\w\u4e00-\u9fff\-]", "-", title) + sanitized = re.sub(r"-{2,}", "-", sanitized).strip("-") + return sanitized if sanitized else None + + +def handle_session_start(data: dict[str, Any]) -> None: + """处理 SessionStart：预热 inventory，写 start 事件，初始化 AgentLens。""" + current_sid = session_id_of(data) + if not current_sid: + return + paths = build_runtime_paths() + cwd_value = cwd(data) + skills_meta, rules_meta = scanner.scan_skills_and_rules(cwd_value) + collector.cache_inventory(paths.state_path, current_sid, skills_meta, rules_meta) + emitter.emit(paths.log_path, emitter.build_start_event(sid=current_sid, agent="main")) + agentlens.emit_session_start(state_path=paths.state_path, sid=current_sid, cwd=cwd_value, agent="main") + + +def handle_user_prompt_submit(data: dict[str, Any]) -> None: + """处理 UserPromptSubmit：建立 turn 边界，并打开 AgentLens turn。""" + current_sid = session_id_of(data) + if not current_sid: + return + paths = build_runtime_paths() + cwd_value = cwd(data) + turn_id = derive_turn_id(data) + prompt_len = prompt_text_length(data) + prompt_text = extract_prompt_text(data) + business_scenario = derive_business_scenario(prompt_text) + + emitter.emit( + paths.log_path, + emitter.build_prompt_submit_event(sid=current_sid, turn_id=turn_id, agent="main", prompt_len=prompt_len), + ) + agentlens.emit_turn_start( + state_path=paths.state_path, + sid=current_sid, + cwd=cwd_value, + turn_id=turn_id, + prompt_meta={"prompt_len": prompt_len} if prompt_len is not None else None, + business_scenario=business_scenario, + ) + + +def handle_pre(data: dict[str, Any]) -> None: + """处理 PreToolUse：仅记录 pending，等待 post 配对。""" + collector.record_pre(build_runtime_paths().pending_path, data) + + +def handle_post(data: dict[str, Any]) -> None: + """处理 PostToolUse：计算耗时、归因 agent、发 tool/usage 事件。""" + current_sid = session_id_of(data) + if not current_sid: + return + paths = build_runtime_paths() + cwd_value = cwd(data) + duration_ms = collector.record_post(paths.pending_path, data) + + skills_meta, rules_meta = collector.load_cached_inventory(paths.state_path, current_sid) + if not skills_meta and not rules_meta: + skills_meta, rules_meta = scanner.scan_skills_and_rules(cwd_value) + collector.cache_inventory(paths.state_path, current_sid, skills_meta, rules_meta) + + active_agent, _ = agent_identity.resolve_active_agent_for_event( + state_path=paths.state_path, + sid=current_sid, + cwd=cwd_value, + data=data, + ) + used_skills, used_rules = collector.record_tool_usage( + state_path=paths.state_path, + sid=current_sid, + data=data, + skills_meta=skills_meta, + rules_meta=rules_meta, + active_agent=active_agent, + collect_skills=True, + ) + + tool_name = str(data.get("tool_name") or "") + transcript_path = collector.transcript_path(data) + turn_id = current_turn_id(paths.state_path, current_sid) + event_ts = time.time() + tool_call_id = collector.extract_tool_call_id(data) + dv_ctx = resolve_devflow(paths.state_path, current_sid, cwd_value, paths.log_path) + task_slug = str(dv_ctx.get("task_slug") or "").strip() if dv_ctx else None + stage = str(dv_ctx.get("current_stage") or "").strip() if dv_ctx else None + flush_pending_tool_events( + state_path=paths.state_path, + log_path=paths.log_path, + sid=current_sid, + ) + tool_context = collector.find_current_tool_context( + state_path=paths.state_path, + sid=current_sid, + transcript_path=transcript_path, + tool_name=tool_name, + call_id=tool_call_id, + event_ts=event_ts, + claim=True, + ) + duplicate_tool_context = bool(isinstance(tool_context, dict) and tool_context.get("duplicate")) + if duplicate_tool_context: + tool_context = None + if isinstance(tool_context, dict): + resolved_transcript_path = str(tool_context.get("transcript_path") or "").strip() + if resolved_transcript_path: + transcript_path = resolved_transcript_path + resolved_agent = str(tool_context.get("agent") or "").strip() + if resolved_agent: + active_agent = resolved_agent + + tool_pt_id = str(tool_context.get("pt_id") or "").strip() if isinstance(tool_context, dict) else None + tool_event = emitter.build_tool_event( + sid=current_sid, + tool=tool_name, + duration_ms=duration_ms, + active_agent=active_agent, + transcript_path=transcript_path, + turn_id=turn_id, + task_slug=task_slug, + stage=stage, + pt_id=tool_pt_id, + ) + tool_event["ts"] = event_ts + tool_event["skill"] = used_skills + tool_event["rule"] = used_rules + tool_event["cwd"] = cwd_value + if tool_call_id: + tool_event["call_id"] = tool_call_id + if isinstance(tool_context, dict): + tool_message_id = str(tool_context.get("message_id") or "").strip() or None + if tool_message_id: + tool_event["message_id"] = tool_message_id + tool_details = tool_context.get("tool_details") + if isinstance(tool_details, dict) and tool_details: + tool_event["tool_details"] = dict(tool_details) + + usage_events = emit_transcript_events( + state_path=paths.state_path, + log_path=paths.log_path, + sid=current_sid, + transcript_paths=collector.related_transcript_paths(current_sid, transcript_path), + current_transcript_path=transcript_path, + active_agent=active_agent, + tool_name=tool_name, + event_kind="usage", + max_lines=80, + turn_id=turn_id, + task_slug=task_slug, + stage=stage, + ) + if duplicate_tool_context: + if not usage_events: + return + agentlens.emit_post_step( + state_path=paths.state_path, + sid=current_sid, + tool_event=None, + usage_events=usage_events, + ) + return + if str(tool_event.get("message_id") or "").strip(): + emitter.emit(paths.log_path, tool_event) + agentlens.emit_post_step( + state_path=paths.state_path, + sid=current_sid, + tool_event=tool_event, + usage_events=usage_events, + ) + return + + fallback_usage = collector.find_fallback_usage_event(tool_event, usage_events) + fallback_usage_mid = str((fallback_usage or {}).get("message_id") or "").strip() or None + if fallback_usage_mid: + tool_event["message_id"] = fallback_usage_mid + emitter.emit(paths.log_path, tool_event) + agentlens.emit_post_step( + state_path=paths.state_path, + sid=current_sid, + tool_event=tool_event, + usage_events=usage_events, + ) + return + + st.append_pending_tool_emit(paths.state_path, current_sid, tool_event) + + +def handle_stop(data: dict[str, Any]) -> None: + """处理 Stop：补发 transcript 事件、汇总 session 成本并关闭 AgentLens。""" + current_sid = session_id_of(data) + if not current_sid: + return + paths = build_runtime_paths() + cwd_value = cwd(data) + active_agent, _ = agent_identity.resolve_active_agent_for_event( + state_path=paths.state_path, + sid=current_sid, + cwd=cwd_value, + data=data, + ) + transcript_path = collector.transcript_path(data) + turn_id = current_turn_id(paths.state_path, current_sid) + dv_ctx = resolve_devflow(paths.state_path, current_sid, cwd_value, paths.log_path) + task_slug = str(dv_ctx.get("task_slug") or "").strip() if dv_ctx else None + stage = str(dv_ctx.get("current_stage") or "").strip() if dv_ctx else None + stop_events = emit_transcript_events( + state_path=paths.state_path, + log_path=paths.log_path, + sid=current_sid, + transcript_paths=collector.settled_related_transcript_paths(current_sid, transcript_path), + current_transcript_path=transcript_path, + active_agent=active_agent, + tool_name="__stop__", + event_kind="stop", + max_lines=200, + cost_session=emitter.sum_session_cost(paths.log_path, current_sid), + turn_id=turn_id, + task_slug=task_slug, + stage=stage, + ) + flush_pending_tool_events( + state_path=paths.state_path, + log_path=paths.log_path, + sid=current_sid, + force_emit_fallback=True, + ) + agentlens.emit_session_stop( + state_path=paths.state_path, + sid=current_sid, + cwd=cwd_value, + active_agent=active_agent, + transcript_path=transcript_path, + stop_events=stop_events, + ) + + +def resolve_devflow(state_path: Path, sid: str, cwd_value: str, log_path: Path) -> dict[str, Any] | None: + """解析 devflow 上下文，并把本次观测到的 stage 变更立即发成 `stage_transition` 事件。 + + 任何异常都吞掉、返回 None——devflow 感知是可选增强，绝不能让不跑 devflow 的 + 项目或 workflow-state.json 格式变化导致 hook 报错。 + """ + try: + dv_ctx = devflow.resolve_and_diff(state_path, sid, cwd_value) + except Exception: + return None + if not isinstance(dv_ctx, dict): + return None + task_slug = str(dv_ctx.get("task_slug") or "").strip() + if not task_slug: + return None + for change in dv_ctx.get("changes") or []: + if not isinstance(change, dict): + continue + event = emitter.build_stage_transition_event( + sid=sid, + task_slug=task_slug, + stage=str(change.get("stage") or ""), + status=change.get("status"), + executor=change.get("executor"), + retry_count=int(change.get("retry_count") or 0), + review_result=change.get("review_result"), + ) + emitter.emit(log_path, event) + return dv_ctx + + +def current_turn_id(state_path: Path, sid: str) -> str | None: + """从 state 中读取当前 session 激活中的 turn id。""" + try: + turn = st.get_current_turn(state_path, sid) + except Exception: + turn = None + if isinstance(turn, dict): + raw = turn.get("turn_id") + if isinstance(raw, str) and raw.strip(): + return raw.strip() + return None + + +def flush_pending_tool_events( + *, + state_path: Path, + log_path: Path, + sid: str, + force_emit_fallback: bool = False, +) -> None: + """强制 flush 延迟的 tool 事件，常用于 stop/replay 路径。""" + pending = st.get_pending_tool_emits(state_path, sid) + if not pending: + return + + remaining: list[dict[str, Any]] = [] + for tool_event in pending: + tool_name = str(tool_event.get("tool") or "").strip() + transcript_path = str(tool_event.get("transcript_path") or "").strip() + call_id = str(tool_event.get("call_id") or "").strip() or None + event_ts = tool_event.get("ts") + try: + event_ts_float = float(event_ts) if event_ts is not None else None + except Exception: + event_ts_float = None + + tool_context = collector.find_current_tool_context( + state_path=state_path, + sid=sid, + transcript_path=transcript_path, + tool_name=tool_name, + call_id=call_id, + event_ts=event_ts_float, + claim=True, + ) + if isinstance(tool_context, dict) and tool_context.get("duplicate"): + continue + if isinstance(tool_context, dict): + resolved_transcript_path = str(tool_context.get("transcript_path") or "").strip() + if resolved_transcript_path: + tool_event["transcript_path"] = resolved_transcript_path + resolved_agent = str(tool_context.get("agent") or "").strip() + if resolved_agent: + tool_event["agent"] = resolved_agent + tool_message_id = str(tool_context.get("message_id") or "").strip() or None + if tool_message_id: + tool_event["message_id"] = tool_message_id + tool_details = tool_context.get("tool_details") + if isinstance(tool_details, dict) and tool_details: + tool_event["tool_details"] = dict(tool_details) + emitter.emit(log_path, tool_event) + agentlens.emit_post_step( + state_path=state_path, + sid=sid, + tool_event=tool_event, + usage_events=[], + ) + continue + if force_emit_fallback: + emitter.emit(log_path, tool_event) + agentlens.emit_post_step( + state_path=state_path, + sid=sid, + tool_event=tool_event, + usage_events=[], + ) + continue + remaining.append(tool_event) + + st.replace_pending_tool_emits(state_path, sid, remaining) + + +def emit_transcript_events( + *, + state_path: Path, + log_path: Path, + sid: str, + transcript_paths: list[str], + current_transcript_path: str, + active_agent: str, + tool_name: str, + event_kind: str, + max_lines: int, + cost_session: float | None = None, + turn_id: str | None = None, + task_slug: str | None = None, + stage: str | None = None, +) -> list[dict[str, Any]]: + """发出基于 transcript 增量重建得到的 usage/stop 事件。""" + emitted_events: list[dict[str, Any]] = [] + current_resolved = str(Path(current_transcript_path).resolve()) if current_transcript_path else "" + for path in transcript_paths: + scan = collector.collect_transcript_entries( + state_path=state_path, + sid=sid, + transcript_path=path, + max_lines=max_lines, + ) + entries = scan.get("entries", []) or [] + event_tool = tool_name if event_kind == "usage" and path == current_resolved else "model_request" + event_agent = active_agent if path == current_resolved else agent_identity.agent_for_transcript_path(path, active_agent) + + for idx, entry in enumerate(entries): + event_key = collector.build_transcript_event_key(kind=event_kind, tool=event_tool, entry=entry) + if not st.claim_transcript_event(state_path, sid, event_key, path): + continue + tokens = entry.get("tokens") + model = entry.get("model") + if event_kind == "stop": + event = emitter.build_stop_event( + sid=sid, + tokens=tokens, + model=model, + cost_usd=emitter.cost_of(tokens, model), + cost_session_usd=cost_session if idx == len(entries) - 1 else None, + active_agent=event_agent, + transcript_path=path, + source_offset=int(entry.get("offset", 0) or 0), + turn_id=turn_id, + message_id=str(entry.get("message_id") or "").strip() or None, + task_slug=task_slug, + stage=stage, + ) + else: + event = emitter.build_usage_event( + sid=sid, + tool=event_tool, + tokens=tokens, + active_agent=event_agent, + model=model, + cost_usd=emitter.cost_of(tokens, model), + transcript_path=path, + source_offset=int(entry.get("offset", 0) or 0), + turn_id=turn_id, + message_id=str(entry.get("message_id") or "").strip() or None, + task_slug=task_slug, + stage=stage, + ) + emitter.emit(log_path, event) + emitted_events.append(event) + + st.commit_transcript_progress( + state_path, + sid, + path, + offset=int(scan.get("new_offset", 0) or 0), + last_cumulative_usage=scan.get("last_cumulative"), + ) + return emitted_events + + +def main(argv: list[str]) -> int: + """读取 stdin 输入并按 phase 执行对应 hook 处理分支。""" + phase = normalize_phase(argv[0] if argv else "") + current_sid = "" + try: + data = collector.read_stdin_json() + current_sid = session_id_of(data) + if phase == "session-start": + handle_session_start(data) + elif phase == "user-prompt-submit": + handle_user_prompt_submit(data) + elif phase == "pre": + handle_pre(data) + elif phase == "post": + handle_post(data) + elif phase == "stop": + handle_stop(data) + else: + emitter.emit( + build_runtime_paths().log_path, + emitter.build_error_event(phase=phase or "unknown", error=f"Unsupported phase: {phase!r}", sid=current_sid), + ) + except Exception as exc: + emit_error(phase=phase or "unknown", sid=current_sid, err=exc) + return 0 diff --git a/.claude/skills/agent-observability/scripts/core/scanner.py b/.claude/skills/agent-observability/scripts/core/scanner.py new file mode 100644 index 0000000..86b5305 --- /dev/null +++ b/.claude/skills/agent-observability/scripts/core/scanner.py @@ -0,0 +1,496 @@ +"""Skill / Rule 静态扫描层。 + +这一层只负责扫描项目里的 `.codebuddy/skills` 与 `.codebuddy/rules`： +- 识别有哪些可用 skill / rule +- 提取它们的基础元数据与 frontmatter +- 提供 path-based 命中推断依赖的 inventory 信息 + +它不依赖具体 workflow 语义，本身是通用层。 +""" +from __future__ import annotations + +import os +import re +from pathlib import Path +from typing import Any + +# --- 用于解析 SKILL.md / RULE.mdc frontmatter 的正则 --- +FRONTMATTER_RE = re.compile(r"^---\s*\n(.*?)\n---", re.S) +NAME_FIELD_RE = re.compile(r"^name:\s*(.+)$", re.M) +VERSION_FIELD_RE = re.compile(r"^version:\s*(.+)$", re.M) +AUTHOR_FIELD_RE = re.compile(r"^author:\s*(.+)$", re.M) +TAGS_FIELD_RE = re.compile(r"^tags:\s*\[([^\]]+)\]", re.M) +DESC_FIELD_RE = re.compile(r"^description:\s*(.+?)(?=\n\w+:|\n---|\Z)", re.S | re.M) +ALWAYS_APPLY_RE = re.compile(r"^alwaysApply:\s*(true|false)\s*$", re.M | re.I) +ENABLED_RE = re.compile(r"^enabled:\s*(true|false)\s*$", re.M | re.I) +GLOBS_RE = re.compile(r"^globs:\s*(.+?)(?=\n\w+:|\n---|\Z)", re.S | re.M) +APPLY_AGENTS_RE = re.compile(r"^applyAgents:\s*(.+?)(?=\n\w+:|\n---|\Z)", re.S | re.M) +EXCLUDE_AGENTS_RE = re.compile(r"^excludeAgents:\s*(.+?)(?=\n\w+:|\n---|\Z)", re.S | re.M) + +# 固定的 inventory 根目录（仅限项目内） +PROJECT_SKILLS_REL = Path(".codebuddy/skills") +PROJECT_RULES_REL = Path(".codebuddy/rules") + +# 支持识别的 skill 子模块根目录 +SUBMODULE_DIRS = {"references", "scripts", "templates", "examples", "checklists", "roles"} + +# 允许严格提取的路径类字段键名（不扫描自由文本） +PATH_VALUE_KEYS = { + "file_path", "filePath", "path", "target_file", "target_directory", + "cwd", "rule", "rule_path", "skill_path", +} +PATH_LIST_KEYS = {"paths", "files", "rules", "skills"} + +# 扫描时要跳过的目录 +SKIP_DIR_PARTS = {"skills-by-node", "node_modules", ".git", "__pycache__", "dist"} + +# 可能触发 skill 的工具名 +USE_SKILL_TOOLS = {"use_skill", "UseSkill", "load_skill", "Skill", "skill"} + + +def _is_skipped_path(p: Path) -> bool: + """判断路径是否命中扫描时应跳过的目录集合。""" + parts = set(p.parts) + return bool(parts & SKIP_DIR_PARTS) + + +def _parse_frontmatter(path: Path) -> dict[str, Any]: + """解析 `SKILL.md` 或 `RULE.mdc` 中的 frontmatter。""" + try: + text = path.read_text("utf-8", errors="ignore") + except Exception: + return {} + m = FRONTMATTER_RE.match(text) + if not m: + return {} + block = m.group(1) + info: dict = {} + + def _grab(regex, key, post=None): + mm = regex.search(block) + if mm: + v = mm.group(1).strip().strip('"\'').strip() + info[key] = post(v) if post else v + + _grab(NAME_FIELD_RE, "name") + _grab(VERSION_FIELD_RE, "version") + _grab(AUTHOR_FIELD_RE, "author") + _grab(TAGS_FIELD_RE, "tags", + lambda v: [t.strip().strip('"\'') for t in v.split(",") if t.strip()]) + _grab(DESC_FIELD_RE, "description", lambda v: v.strip().replace("\n", " ")[:200]) + return info + + +def _parse_list_field(block: str, regex: re.Pattern[str]) -> list[str]: + """从 frontmatter 文本块中解析一个逗号分隔的列表字段。""" + mm = regex.search(block) + if not mm: + return [] + raw = mm.group(1).strip().strip('"\'') + if raw.startswith("["): + raw = raw.strip("[]") + return [g.strip().strip('"\'').lower() for g in raw.split(",") if g.strip()] + + +def _parse_rule_frontmatter(rule_path: Path) -> dict[str, Any]: + """解析 `RULE.mdc` 中和规则生效相关的 frontmatter 字段。""" + try: + text = rule_path.read_text("utf-8", errors="ignore") + except Exception: + return {} + m = FRONTMATTER_RE.match(text) + if not m: + return {} + block = m.group(1) + info: dict = {} + + mm = ALWAYS_APPLY_RE.search(block) + if mm: + info["alwaysApply"] = mm.group(1).lower() == "true" + mm = ENABLED_RE.search(block) + if mm: + info["enabled"] = mm.group(1).lower() == "true" + globs = _parse_list_field(block, GLOBS_RE) + if globs: + info["globs"] = globs + + apply_agents = _parse_list_field(block, APPLY_AGENTS_RE) + if apply_agents: + info["applyAgents"] = apply_agents + + exclude_agents = _parse_list_field(block, EXCLUDE_AGENTS_RE) + if exclude_agents: + info["excludeAgents"] = exclude_agents + + mm = DESC_FIELD_RE.search(block) + if mm: + info["description"] = mm.group(1).strip().strip('"\'').replace("\n", " ")[:200] + return info + + +def _classify_source(path: Path, cwd: Path | None) -> str: + """识别来源类型；当前只支持项目内 `.codebuddy` 路径。""" + if not cwd: + return "unknown" + try: + path_str = str(path.resolve()) + cwd_str = str(cwd.resolve()) + except Exception: + return "unknown" + if path_str.startswith(cwd_str + "/.codebuddy/"): + return "project" + return "unknown" + + +def _collect_submodules(skill_dir: Path) -> dict[str, list]: + """收集一个 skill 内部的子模块目录，如 references/scripts/templates。""" + sub: dict = {} + if not skill_dir.is_dir(): + return sub + for kind in ("references", "scripts", "templates", "examples", "checklists", "roles"): + d = skill_dir / kind + if d.is_dir(): + files = [str(f.relative_to(skill_dir)) + for f in d.rglob("*") if f.is_file()] + if files: + sub[kind] = sorted(files)[:30] + return sub + + +def scan_skills_and_rules(cwd: str) -> tuple[dict[str, Any], dict[str, Any]]: + """扫描项目固定根目录：`.codebuddy/skills` 与 `.codebuddy/rules`。""" + skills: dict = {} + rules: dict = {} + cwd_path = Path(cwd) if cwd else None + if not cwd_path or not cwd_path.exists(): + return skills, rules + + skills_root = (cwd_path / PROJECT_SKILLS_REL).resolve() + rules_root = (cwd_path / PROJECT_RULES_REL).resolve() + + if skills_root.is_dir() and not _is_skipped_path(skills_root): + for p in skills_root.rglob("SKILL.md"): + if _is_skipped_path(p): + continue + skill_dir = p.parent + name = skill_dir.name + if not name or name in skills: + continue + meta = _parse_frontmatter(p) + skills[name] = { + "source": _classify_source(skill_dir, cwd_path), + "path": str(skill_dir), + "has_skill_md": True, + "version": meta.get("version"), + "author": meta.get("author"), + "description": meta.get("description"), + "submodules": _collect_submodules(skill_dir), + } + + if rules_root.is_dir() and not _is_skipped_path(rules_root): + for p in rules_root.rglob("*.mdc"): + if _is_skipped_path(p): + continue + name = p.parent.name if p.name == "RULE.mdc" else p.stem + if not name or name in rules: + continue + meta = _parse_rule_frontmatter(p) + rules[name] = { + "source": _classify_source(p.parent, cwd_path), + "path": str(p), + "alwaysApply": meta.get("alwaysApply", False), + "enabled": meta.get("enabled", True), + "globs": meta.get("globs") or [], + "applyAgents": meta.get("applyAgents") or [], + "excludeAgents": meta.get("excludeAgents") or [], + "description": meta.get("description"), + } + + return skills, rules + + +def lookup_skill_meta(skills_meta: dict[str, Any], name: str) -> dict[str, Any] | None: + """从缓存的 inventory 字典里按名称查找 skill 元数据。""" + meta = (skills_meta or {}).get(name) + return meta if isinstance(meta, dict) else None + + +def lookup_rule_meta(rules_meta: dict[str, Any], name: str) -> dict[str, Any] | None: + """从缓存的 inventory 字典里按名称查找 rule 元数据。""" + meta = (rules_meta or {}).get(name) + return meta if isinstance(meta, dict) else None + + +def _collect_path_strings(obj: Any) -> list[str]: + """只收集结构化路径字段，绝不解析自由文本。""" + out: list[str] = [] + + def _walk(x: Any) -> None: + if isinstance(x, dict): + for k, v in x.items(): + if k in PATH_VALUE_KEYS and isinstance(v, str) and v.strip(): + out.append(v.strip()) + elif k in PATH_LIST_KEYS: + if isinstance(v, str) and v.strip(): + out.append(v.strip()) + elif isinstance(v, list): + out.extend([ + s.strip() for s in v + if isinstance(s, str) and s.strip() + ]) + elif isinstance(v, (dict, list)): + _walk(v) + elif isinstance(x, list): + for it in x: + if isinstance(it, (dict, list)): + _walk(it) + + _walk(obj) + return list(dict.fromkeys(out)) + + +def _path_parts(path_str: str) -> list[str]: + """把路径拆成标准化片段列表，便于后续命中判断。""" + p = path_str.replace("\\", "/").strip() + return [part for part in p.split("/") if part] + + +def _extract_skill_from_path(path_str: str) -> str | None: + """从路径中抽取被引用的 skill 名。""" + parts = _path_parts(path_str) + for i in range(len(parts) - 2): + if parts[i] == ".codebuddy" and parts[i + 1] == "skills": + name = parts[i + 2] + if name and re.fullmatch(r"[a-zA-Z0-9_\-]+", name): + return name + return None + + +def _extract_rule_from_path(path_str: str) -> str | None: + """从路径中抽取被引用的 rule 名。""" + parts = _path_parts(path_str) + for i in range(len(parts) - 2): + if parts[i] == ".codebuddy" and parts[i + 1] == "rules": + name = parts[i + 2] + if name.lower().endswith(".mdc"): + name = Path(name).stem + if name and re.fullmatch(r"[a-zA-Z0-9_\-]+", name): + return name + return None + + +def _is_rule_allowed_for_agent(name: str, meta: dict[str, Any], active_agent: str | None) -> bool: + """按角色判断 rule 是否可用；`global` 永远保持可选。""" + if str(name).strip().lower() == "global": + return True + + agent = str(active_agent or "").strip().lower() + if not agent: + return True + + apply_agents = [str(a).strip().lower() for a in (meta.get("applyAgents") or []) if str(a).strip()] + exclude_agents = [str(a).strip().lower() for a in (meta.get("excludeAgents") or []) if str(a).strip()] + + if apply_agents and agent not in apply_agents: + return False + if exclude_agents and agent in exclude_agents: + return False + return True + + +def _glob_matches_any_path(globs: list[str], paths_to_check: list[str]) -> bool: + """判断配置的 glob 是否命中任一收集到的路径。""" + if not globs or not paths_to_check: + return False + import fnmatch + + for p in paths_to_check: + basename = os.path.basename(p) + if any(fnmatch.fnmatch(p, g) or fnmatch.fnmatch(basename, g) for g in globs): + return True + return False + + +def evaluate_rules_for_call( + data: dict, + rules_meta: dict, + active_agent: str | None = None, +) -> dict[str, dict[str, Any]]: + """评估当前工具调用下所有已知 rule，并保留未命中的原因。""" + if not rules_meta: + return {} + + tool_input = data.get("tool_input") or {} + paths_to_check = _collect_path_strings(tool_input) + _, path_rules = extract_paths_from_tool_call(data) + path_rule_set = set(path_rules) + + evaluations: dict[str, dict[str, Any]] = {} + for name, meta in (rules_meta or {}).items(): + if not isinstance(meta, dict): + continue + + globs = [str(g).strip() for g in (meta.get("globs") or []) if str(g).strip()] + apply_agents = [str(a).strip().lower() for a in (meta.get("applyAgents") or []) if str(a).strip()] + exclude_agents = [str(a).strip().lower() for a in (meta.get("excludeAgents") or []) if str(a).strip()] + evaluation: dict[str, Any] = {"matched": False} + + if name in path_rule_set: + evaluation["matched"] = True + evaluation["via"] = "path_inferred" + elif meta.get("enabled") is False: + evaluation["reason"] = "disabled" + elif not _is_rule_allowed_for_agent(name, meta, active_agent): + evaluation["reason"] = "agent_filtered" + if apply_agents: + evaluation["applyAgents"] = apply_agents + if exclude_agents: + evaluation["excludeAgents"] = exclude_agents + elif meta.get("alwaysApply"): + evaluation["matched"] = True + evaluation["via"] = "always_apply" + elif globs: + evaluation["globs"] = globs + evaluation["paths_checked"] = len(paths_to_check) + if not paths_to_check: + evaluation["reason"] = "no_paths" + elif _glob_matches_any_path(globs, paths_to_check): + evaluation["matched"] = True + evaluation["via"] = "glob" + else: + evaluation["reason"] = "glob_not_matched" + evaluation["paths_sample"] = paths_to_check[:3] + else: + evaluation["reason"] = "no_globs" + + evaluations[name] = evaluation + + return evaluations + + +def unmatched_rule_diagnostics_for_call( + data: dict, + rules_meta: dict, + active_agent: str | None = None, +) -> dict[str, dict[str, Any]]: + """返回当前工具调用下未命中的 rule 的精简诊断信息。""" + diagnostics: dict[str, dict[str, Any]] = {} + for name, evaluation in evaluate_rules_for_call(data, rules_meta, active_agent=active_agent).items(): + if evaluation.get("matched"): + continue + item: dict[str, Any] = {"reason": evaluation.get("reason") or "unknown"} + for key in ("globs", "paths_checked", "paths_sample", "applyAgents", "excludeAgents"): + value = evaluation.get(key) + if value is not None and value != []: + item[key] = value + diagnostics[name] = item + return diagnostics + + +def filtered_rules_for_agent(rules_meta: dict, active_agent: str | None = None) -> list[str]: + """列出因 agent 过滤而被排除的 rule（仅看已启用项）。""" + filtered: list[str] = [] + for name, evaluation in evaluate_rules_for_call({}, rules_meta, active_agent=active_agent).items(): + if evaluation.get("reason") == "agent_filtered": + filtered.append(name) + return sorted(set(filtered)) + + +def active_rules_for_call(data: dict, rules_meta: dict, active_agent: str | None = None) -> list[str]: + """判断给定工具调用下哪些 rule 处于生效状态。""" + active: list[str] = [] + for name, evaluation in evaluate_rules_for_call(data, rules_meta, active_agent=active_agent).items(): + if evaluation.get("matched") and evaluation.get("via") != "path_inferred": + active.append(name) + return sorted(set(active)) + + +def extract_skill_from_tool_call(data: dict) -> str | None: + """从类似 use_skill 的工具调用里提取 skill 名称。""" + tool = data.get("tool_name", "") + if tool not in USE_SKILL_TOOLS: + return None + tool_input = data.get("tool_input") or {} + if isinstance(tool_input, dict): + return tool_input.get("command") or tool_input.get("name") or tool_input.get("skill") + return None + + +def is_brainstorming_call(data: dict) -> bool: + """判断当前工具调用是否正在激活或使用 brainstorming skill。""" + direct = extract_skill_from_tool_call(data) + if isinstance(direct, str) and direct.strip().lower() == "brainstorming": + return True + + path_skills, _ = extract_paths_from_tool_call(data) + return any(str(s).strip().lower() == "brainstorming" for s in path_skills) + + +def extract_paths_from_tool_call(data: dict) -> tuple[list[str], list[str]]: + """只根据 `.codebuddy` 根目录下的结构化路径推断 skill/rule。""" + tool_input = data.get("tool_input") or {} + if not isinstance(tool_input, dict): + return [], [] + + skills: list[str] = [] + rules: list[str] = [] + for p in _collect_path_strings(tool_input): + s = _extract_skill_from_path(p) + if s: + skills.append(s) + r = _extract_rule_from_path(p) + if r: + rules.append(r) + + return list(dict.fromkeys(skills)), list(dict.fromkeys(rules)) + + +def extract_submodule_hits(data: dict) -> list[dict[str, str]]: + """仅从结构化 skill 路径中提取子模块命中结果，不扫描自由文本。""" + tool_input = data.get("tool_input") or {} + if not isinstance(tool_input, dict): + return [] + + seen: set[tuple[str, str]] = set() + hits: list[dict[str, str]] = [] + path_values = _collect_path_strings(tool_input) + + for p in path_values: + parts = _path_parts(p) + for i in range(len(parts) - 3): + if parts[i] == ".codebuddy" and parts[i + 1] == "skills": + skill_name = parts[i + 2] + sub_root = parts[i + 3] + if not re.fullmatch(r"[a-zA-Z0-9_\-]+", skill_name): + continue + if sub_root not in SUBMODULE_DIRS: + continue + sub_path = "/".join(parts[i + 3:]).rstrip("/") + key = (skill_name, sub_path) + if key in seen: + continue + seen.add(key) + hits.append({"skill": skill_name, "submodule": sub_path}) + break + + return hits + + +def extract_bash_skill_scripts(data: dict) -> list[str]: + """从命令文本中的显式 `.codebuddy/skills` 路径识别 skill 脚本。""" + if data.get("tool_name") not in ("Bash", "bash", "execute_command"): + return [] + tool_input = data.get("tool_input") or {} + if not isinstance(tool_input, dict): + return [] + cmd = tool_input.get("command") or "" + if not isinstance(cmd, str) or not cmd.strip(): + return [] + + skills: list[str] = [] + for token in cmd.split(): + t = token.strip("\"'`;,()[]{}") + s = _extract_skill_from_path(t) + if s: + skills.append(s) + return list(dict.fromkeys(skills)) diff --git a/.claude/skills/agent-observability/scripts/core/state.py b/.claude/skills/agent-observability/scripts/core/state.py new file mode 100644 index 0000000..5e7d33e --- /dev/null +++ b/.claude/skills/agent-observability/scripts/core/state.py @@ -0,0 +1,1222 @@ +"""持久化状态层。 + +这一层负责跨 hook 调用保存与协调状态： +- session 级 skill/rule 使用统计 +- 当前 agent、派发关系与历史 +- transcript offset、去重 claim、pending emits +- AgentLens sidecar 状态 +""" +from __future__ import annotations + +import json +import time +from contextlib import contextmanager +from pathlib import Path +from typing import Any + +try: + import fcntl +except Exception: # pragma: no cover - non-posix fallback + fcntl = None # type: ignore + + +def get_state_path(base_dir: Path) -> Path: + """返回主状态文件路径。""" + log_dir = base_dir / "logs" + log_dir.mkdir(parents=True, exist_ok=True) + return log_dir / ".state.json" + + +def get_pending_path(base_dir: Path) -> Path: + """返回 Pre/PostToolUse 配对使用的 pending 文件路径。""" + log_dir = base_dir / "logs" + log_dir.mkdir(parents=True, exist_ok=True) + return log_dir / ".pending_calls.json" + + +def load_state(state_path: Path) -> dict[str, Any]: + """从磁盘加载状态；任何异常都回退为空字典。""" + if state_path.exists(): + try: + return json.loads(state_path.read_text("utf-8")) + except Exception: + return {} + return {} + + +def save_state(state_path: Path, state: dict[str, Any]) -> None: + """把状态写回磁盘。""" + state_path.write_text(json.dumps(state, ensure_ascii=False), encoding="utf-8") + + +def get_state_lock_path(state_path: Path) -> Path: + """返回状态写锁文件路径。""" + return state_path.parent / ".state.lock" + + +def get_transcript_seen_path(state_path: Path) -> Path: + """返回 transcript usage 事件 claim 的 sidecar 台账文件。 + + 这部分单独存放，不混进 `.state.json`，是为了避免并发 hook 写状态时， + 旧 state 覆盖掉已经 claim 过的 transcript offset。 + """ + return state_path.parent / ".transcript_events_seen.json" + + +@contextmanager +def state_lock(state_path: Path): + """为状态修改提供跨进程建议锁（独占）。""" + lock_path = get_state_lock_path(state_path) + lock_path.parent.mkdir(parents=True, exist_ok=True) + with lock_path.open("a+", encoding="utf-8") as fp: + if fcntl is not None: + fcntl.flock(fp.fileno(), fcntl.LOCK_EX) + try: + yield + finally: + if fcntl is not None: + fcntl.flock(fp.fileno(), fcntl.LOCK_UN) + + +@contextmanager +def state_read_lock(state_path: Path): + """为状态一致性读取提供跨进程共享读锁。""" + lock_path = get_state_lock_path(state_path) + lock_path.parent.mkdir(parents=True, exist_ok=True) + with lock_path.open("a+", encoding="utf-8") as fp: + if fcntl is not None: + fcntl.flock(fp.fileno(), fcntl.LOCK_SH) + try: + yield + finally: + if fcntl is not None: + fcntl.flock(fp.fileno(), fcntl.LOCK_UN) + + +def update_state_locked(state_path: Path, updater): + """在跨进程锁保护下完成 load -> mutate -> save。""" + with state_lock(state_path): + state = load_state(state_path) + result = updater(state) + save_state(state_path, state) + return result + + +def load_pending(pending_path: Path) -> dict[str, Any]: + """加载待配对的 pre/post 数据。""" + if pending_path.exists(): + try: + return json.loads(pending_path.read_text("utf-8")) + except Exception: + return {} + return {} + + +def save_pending(pending_path: Path, data: dict[str, Any]) -> None: + """保存待配对的 pre/post 数据。""" + pending_path.write_text(json.dumps(data), encoding="utf-8") + + +def ensure_session(state: dict, sid: str) -> dict: + """确保某个 session 的状态结构存在，并返回该 session 字典。 + + Session 结构： + { + "skills": {name: {count, first_ts, last_ts, tools, via, source, by_agent}}, + "rules": {name: {count, first_ts, last_ts, tools, via, source, by_agent}}, + "started_at": float, + "current_agent": str, + "agent_history": [{ts, agent, evidence}], + "dispatched": {agent_name: count}, + "_skills_meta": {...}, # SessionStart 时缓存的 inventory + "_rules_meta": {...}, + } + """ + if sid not in state: + state[sid] = { + "skills": {}, + "rules": {}, + "started_at": time.time(), + "current_agent": "main", + "agent_history": [], + "dispatched": {}, + } + sess = state[sid] + # 保证字段类型正确，兼容旧数据 + if not isinstance(sess.get("skills"), dict): + sess["skills"] = {} + if not isinstance(sess.get("rules"), dict): + sess["rules"] = {} + sess.setdefault("current_agent", "main") + sess.setdefault("agent_history", []) + sess.setdefault("dispatched", {}) + if not isinstance(sess.get("_pending_tool_emits"), list): + sess["_pending_tool_emits"] = [] + return sess + + +def get_devflow_state(state: dict[str, Any], sid: str) -> dict[str, Any]: + """返回某个 session 缓存的 devflow 上下文与最近一次 stage 快照（见 devflow.py）。""" + sess = state.get(sid) + if not isinstance(sess, dict): + return {} + dv = sess.get("_devflow") + return dv if isinstance(dv, dict) else {} + + +def set_devflow_state(state: dict[str, Any], sid: str, updates: dict[str, Any]) -> None: + """合并写入某个 session 的 devflow 缓存字段（`checked` / `context` / `last_snapshot`）。""" + sess = ensure_session(state, sid) + dv = sess.get("_devflow") + if not isinstance(dv, dict): + dv = {} + sess["_devflow"] = dv + dv.update(updates) + + +def get_agentlens_session(state: dict[str, Any], sid: str) -> dict[str, Any]: + """返回某个 session 的 AgentLens sidecar payload。""" + sess = ensure_session(state, sid) + payload = sess.setdefault("_agentlens", {}) + if not isinstance(payload, dict): + payload = {} + sess["_agentlens"] = payload + return payload + + +AGENTLENS_SCHEMA_VERSION = 2 +_TURN_HISTORY_MAX = 5 + + +def _migrate_agentlens_v1_to_v2(payload: dict[str, Any], sess: dict[str, Any] | None) -> dict[str, Any]: + """把 v1 版 ``_agentlens`` payload 转成以 turn 为中心的 v2 布局。 + + v1 布局（pre-turn）： + {"carrier": {...}, "enabled": bool, "session_id": "...", "app_name": "..."} + v2 布局（turn-centric）： + { + "version": 2, + "session_id": "...", + "enabled": bool, + "app_name": "...", + "current_turn": {"turn_id": "legacy", "started_at": ..., "carrier": {...}, "subagent_spans": {}, "agent_spans": {}, "step_spans": {}}, + "turn_history": [], + } + + 幂等约束：如果已经带有 ``version``，就原样返回。 + """ + if not isinstance(payload, dict): + return {"version": AGENTLENS_SCHEMA_VERSION, "enabled": False, "current_turn": None, "turn_history": []} + if payload.get("version") == AGENTLENS_SCHEMA_VERSION: + payload.setdefault("current_turn", None) + payload.setdefault("turn_history", []) + current = payload.get("current_turn") + if isinstance(current, dict): + current.setdefault("subagent_spans", {}) + current.setdefault("agent_spans", {}) + current.setdefault("step_spans", {}) + return payload + + migrated: dict[str, Any] = { + "version": AGENTLENS_SCHEMA_VERSION, + "session_id": payload.get("session_id"), + "enabled": bool(payload.get("enabled", False)), + } + if "app_name" in payload: + migrated["app_name"] = payload.get("app_name") + if "last_error" in payload: + migrated["last_error"] = payload.get("last_error") + + legacy_carrier = payload.get("carrier") + if isinstance(legacy_carrier, dict) and legacy_carrier: + started_at = None + if isinstance(sess, dict): + try: + started_at = float(sess.get("started_at") or 0) or None + except Exception: + started_at = None + migrated["current_turn"] = { + "turn_id": "legacy", + "started_at": started_at or time.time(), + "carrier": dict(legacy_carrier), + "subagent_spans": {}, + "agent_spans": {}, + "step_spans": {}, + } + else: + migrated["current_turn"] = None + migrated["turn_history"] = [] + return migrated + + +def load_agentlens_session(state_path: Path, sid: str) -> dict[str, Any]: + """加载某个 session 已持久化的 AgentLens sidecar payload。 + + 这里使用共享读锁，避免并发 ``update_agentlens_session`` 写入时读到撕裂数据。 + 返回值会透明地把 v1 payload 迁移成 v2 副本；真正落盘迁移会在下一次写入时完成。 + """ + with state_read_lock(state_path): + state = load_state(state_path) + sess = state.get(sid) + if not isinstance(sess, dict): + return {} + payload = sess.get("_agentlens") + if not isinstance(payload, dict): + return {} + # 先给调用方返回迁移后的副本，真正持久化迁移留到后续写入时完成。 + return _migrate_agentlens_v1_to_v2(dict(payload), sess) + + +def update_agentlens_session( + state_path: Path, + sid: str, + updates: dict[str, Any], + *, + clear_keys: list[str] | None = None, +) -> dict[str, Any]: + """原子地持久化某个 session 的 AgentLens sidecar 字段。 + + 在合并 updates 之前会先做 v1→v2 迁移，这样磁盘上的 payload 会随着写入自然收敛到 v2。 + """ + + def _update(state: dict[str, Any]) -> dict[str, Any]: + sess = ensure_session(state, sid) + payload = get_agentlens_session(state, sid) + # 如果仍是 v1，就原地迁移。 + if payload.get("version") != AGENTLENS_SCHEMA_VERSION: + migrated = _migrate_agentlens_v1_to_v2(dict(payload), sess) + payload.clear() + payload.update(migrated) + for key in clear_keys or []: + payload.pop(key, None) + for key, value in updates.items(): + if value is None: + payload.pop(key, None) + else: + payload[key] = value + payload["version"] = AGENTLENS_SCHEMA_VERSION + return dict(payload) + + return update_state_locked(state_path, _update) + + +def begin_turn( + state_path: Path, + sid: str, + turn_id: str, + carrier: dict[str, str], + *, + started_at: float | None = None, + max_history: int = _TURN_HISTORY_MAX, +) -> dict[str, Any]: + """在独占锁下打开一个新的 turn。 + + 语义： + - 如果存在 ``current_turn``，先把它归档进有上限的 ``turn_history``。 + - 为新 turn 重置 ``subagent_spans`` / ``agent_spans`` / ``step_spans``。 + - 对相同 ``turn_id`` 保持幂等：重复调用直接返回已有 ``current_turn``， + 不重新生成 trace_id。 + + 返回最终生成的 ``current_turn`` 字典。 + """ + ts = float(started_at if started_at is not None else time.time()) + + def _update(state: dict[str, Any]) -> dict[str, Any]: + sess = ensure_session(state, sid) + payload = get_agentlens_session(state, sid) + if payload.get("version") != AGENTLENS_SCHEMA_VERSION: + migrated = _migrate_agentlens_v1_to_v2(dict(payload), sess) + payload.clear() + payload.update(migrated) + + current = payload.get("current_turn") + # 幂等：如果同一个 turn_id 还在处理中，就原样返回。 + if isinstance(current, dict) and str(current.get("turn_id") or "") == str(turn_id): + return dict(current) + + # 归档上一个 turn。 + if isinstance(current, dict) and current.get("turn_id"): + history = payload.setdefault("turn_history", []) + if not isinstance(history, list): + history = [] + payload["turn_history"] = history + archived = { + "turn_id": current.get("turn_id"), + "started_at": current.get("started_at"), + "ended_at": ts, + "carrier_traceparent": (current.get("carrier") or {}).get("traceparent"), + } + history.append(archived) + if len(history) > max_history: + del history[: len(history) - max_history] + + new_turn = { + "turn_id": str(turn_id), + "started_at": ts, + "carrier": dict(carrier or {}), + "subagent_spans": {}, + "agent_spans": {}, + "step_spans": {}, + } + payload["current_turn"] = new_turn + payload["version"] = AGENTLENS_SCHEMA_VERSION + return dict(new_turn) + + return update_state_locked(state_path, _update) + + +def upsert_subagent_span( + state_path: Path, + sid: str, + role: str, + carrier: dict[str, str], +) -> dict[str, str]: + """在当前 turn 下登记 subagent 的 ``invoke_agent`` 子 span carrier。 + + 对 `(turn, role)` 维度保持幂等：如果活跃 turn 下已经记录了该角色的 carrier， + 就直接返回已有值，不再重复创建。 + """ + now = time.time() + + def _update(state: dict[str, Any]) -> dict[str, str]: + payload = get_agentlens_session(state, sid) + if payload.get("version") != AGENTLENS_SCHEMA_VERSION: + migrated = _migrate_agentlens_v1_to_v2(dict(payload), state.get(sid) if isinstance(state.get(sid), dict) else None) + payload.clear() + payload.update(migrated) + current = payload.get("current_turn") + if not isinstance(current, dict): + # 没有活跃 turn，无法登记 subagent span。 + return {} + subs = current.setdefault("subagent_spans", {}) + if not isinstance(subs, dict): + subs = {} + current["subagent_spans"] = subs + existing = subs.get(role) + if isinstance(existing, dict) and isinstance(existing.get("carrier"), dict) and existing["carrier"]: + return dict(existing["carrier"]) + subs[role] = {"carrier": dict(carrier or {}), "opened_at": now} + payload["version"] = AGENTLENS_SCHEMA_VERSION + return dict(carrier or {}) + + return update_state_locked(state_path, _update) + + +def upsert_subagent_span_atomic( + state_path: Path, + sid: str, + role: str, + *, + carrier_factory: Any, +) -> dict[str, str]: + """在写锁内原子地获取或创建 subagent 的 ``invoke_agent`` carrier。 + + `carrier_factory` 是一个无参可调用对象，用来生成 carrier 字典。 + 只有在 span 还不存在时才会被调用，从而避免这样的 TOCTOU 竞争： + ``_generate_subagent_carrier`` 先产生了不可逆的 zhiyanllm ``invoke_agent`` span， + 随后另一个 hook 进程已经写入同一角色，导致本次创建变成孤儿。 + """ + now = time.time() + + def _update(state: dict[str, Any]) -> dict[str, str]: + payload = get_agentlens_session(state, sid) + if payload.get("version") != AGENTLENS_SCHEMA_VERSION: + migrated = _migrate_agentlens_v1_to_v2(dict(payload), state.get(sid) if isinstance(state.get(sid), dict) else None) + payload.clear() + payload.update(migrated) + current = payload.get("current_turn") + if not isinstance(current, dict): + return {} + subs = current.setdefault("subagent_spans", {}) + if not isinstance(subs, dict): + subs = {} + current["subagent_spans"] = subs + existing = subs.get(role) + if isinstance(existing, dict) and isinstance(existing.get("carrier"), dict) and existing["carrier"]: + return dict(existing["carrier"]) + new_carrier = carrier_factory() + if not isinstance(new_carrier, dict) or not new_carrier.get("traceparent"): + return {} + subs[role] = {"carrier": dict(new_carrier), "opened_at": now} + payload["version"] = AGENTLENS_SCHEMA_VERSION + return dict(new_carrier) + + return update_state_locked(state_path, _update) + + +def get_current_turn(state_path: Path, sid: str) -> dict[str, Any] | None: + """返回当前活跃的 turn 字典；如果没有打开 turn，就返回 None。""" + payload = load_agentlens_session(state_path, sid) + current = payload.get("current_turn") + return current if isinstance(current, dict) else None + + +def get_subagent_span_carrier(state_path: Path, sid: str, role: str) -> dict[str, str] | None: + """返回活跃 turn 下已持久化的 subagent span carrier；如果没有则返回 None。""" + turn = get_current_turn(state_path, sid) + if not isinstance(turn, dict): + return None + subs = turn.get("subagent_spans") + if not isinstance(subs, dict): + return None + rec = subs.get(role) + if not isinstance(rec, dict): + return None + carrier = rec.get("carrier") + return dict(carrier) if isinstance(carrier, dict) and carrier else None + + +def _empty_agent_aggregate() -> dict[str, Any]: + """创建一份空的 agent 聚合指标骨架。""" + return { + "tokens": { + "input": 0, + "output": 0, + "cache_read": 0, + "cache_creation": 0, + "total": 0, + }, + "event_count": 0, + "cost_usd": 0.0, + "tool_duration_ms": 0, + "llm_call_count": 0, + "tool_call_count": 0, + "step_count": 0, + } + + +def _normalize_agent(agent: str | None) -> str: + """把 agent 名归一化，空值统一落到 ``main``。""" + return str(agent or "main").strip() or "main" + + +def _ensure_turn_agent_spans(current: dict[str, Any]) -> dict[str, Any]: + """确保当前 turn 下存在 `agent_spans` 容器。""" + agent_spans = current.setdefault("agent_spans", {}) + if not isinstance(agent_spans, dict): + agent_spans = {} + current["agent_spans"] = agent_spans + return agent_spans + + +def _ensure_agent_span_rec(current: dict[str, Any], agent: str) -> dict[str, Any]: + """确保指定 agent 在当前 turn 下有一条完整聚合记录。""" + agent_spans = _ensure_turn_agent_spans(current) + rec = agent_spans.get(agent) + if not isinstance(rec, dict): + rec = {} + agent_spans[agent] = rec + carrier = rec.get("carrier") + if not isinstance(carrier, dict): + rec["carrier"] = {} + aggregate = rec.get("aggregate") + if not isinstance(aggregate, dict): + rec["aggregate"] = _empty_agent_aggregate() + return rec + + +def get_agent_span_carrier(state_path: Path, sid: str, agent: str) -> dict[str, str] | None: + """返回活跃 turn 下已持久化的 agent 分组 span carrier；如果没有则返回 None。""" + turn = get_current_turn(state_path, sid) + if not isinstance(turn, dict): + return None + agent_spans = turn.get("agent_spans") + if not isinstance(agent_spans, dict): + return None + rec = agent_spans.get(_normalize_agent(agent)) + if not isinstance(rec, dict): + return None + carrier = rec.get("carrier") + return dict(carrier) if isinstance(carrier, dict) and carrier else None + + +def upsert_agent_span_atomic( + state_path: Path, + sid: str, + agent: str, + *, + carrier_factory: Any, +) -> dict[str, str]: + """原子地获取或创建 agent 分组 span carrier。""" + now = time.time() + normalized_agent = _normalize_agent(agent) + + def _update(state: dict[str, Any]) -> dict[str, str]: + payload = get_agentlens_session(state, sid) + if payload.get("version") != AGENTLENS_SCHEMA_VERSION: + migrated = _migrate_agentlens_v1_to_v2( + dict(payload), + state.get(sid) if isinstance(state.get(sid), dict) else None, + ) + payload.clear() + payload.update(migrated) + current = payload.get("current_turn") + if not isinstance(current, dict): + return {} + rec = _ensure_agent_span_rec(current, normalized_agent) + existing = rec.get("carrier") + if isinstance(existing, dict) and existing.get("traceparent"): + return dict(existing) + new_carrier = carrier_factory() + if not isinstance(new_carrier, dict) or not new_carrier.get("traceparent"): + return {} + rec["carrier"] = dict(new_carrier) + rec.setdefault("opened_at", now) + payload["version"] = AGENTLENS_SCHEMA_VERSION + return dict(new_carrier) + + return update_state_locked(state_path, _update) + + +def bump_agent_aggregate( + state_path: Path, + sid: str, + agent: str, + *, + usage_event: dict[str, Any] | None = None, + tool_event: dict[str, Any] | None = None, + step_created: bool = False, +) -> dict[str, Any]: + """增加活跃 turn 下按 agent 聚合的统计指标。""" + normalized_agent = _normalize_agent(agent) + + def _int(value: Any) -> int: + try: + return int(value or 0) + except Exception: + return 0 + + def _float(value: Any) -> float: + try: + return float(value or 0) + except Exception: + return 0.0 + + def _update(state: dict[str, Any]) -> dict[str, Any]: + payload = get_agentlens_session(state, sid) + if payload.get("version") != AGENTLENS_SCHEMA_VERSION: + migrated = _migrate_agentlens_v1_to_v2( + dict(payload), + state.get(sid) if isinstance(state.get(sid), dict) else None, + ) + payload.clear() + payload.update(migrated) + current = payload.get("current_turn") + if not isinstance(current, dict): + return {} + rec = _ensure_agent_span_rec(current, normalized_agent) + aggregate = rec.get("aggregate") + if not isinstance(aggregate, dict): + aggregate = _empty_agent_aggregate() + rec["aggregate"] = aggregate + tokens = aggregate.setdefault("tokens", {}) + if not isinstance(tokens, dict): + tokens = {} + aggregate["tokens"] = tokens + + if step_created: + aggregate["step_count"] = _int(aggregate.get("step_count")) + 1 + + if isinstance(tool_event, dict): + aggregate["tool_call_count"] = _int(aggregate.get("tool_call_count")) + 1 + ms = tool_event.get("ms") + if ms is not None: + aggregate["tool_duration_ms"] = _int(aggregate.get("tool_duration_ms")) + _int(ms) + + if isinstance(usage_event, dict): + raw_tokens = usage_event.get("tokens") + if isinstance(raw_tokens, dict): + # 聚合 agent 摘要里需要暴露的字段。 + for key in ("input", "output", "cache_read", "cache_creation"): + tokens[key] = _int(tokens.get(key)) + _int(raw_tokens.get(key)) + # total = input + output，其中 input 已经包含 cache_read。 + tokens["total"] = _int(tokens.get("input")) + _int(tokens.get("output")) + aggregate["llm_call_count"] = _int(aggregate.get("llm_call_count")) + 1 + aggregate["event_count"] = _int(aggregate.get("event_count")) + 1 + aggregate["cost_usd"] = round(_float(aggregate.get("cost_usd")) + _float(usage_event.get("cost_usd")), 10) + + payload["version"] = AGENTLENS_SCHEMA_VERSION + return json.loads(json.dumps(aggregate, ensure_ascii=False)) + + return update_state_locked(state_path, _update) + + +def get_subagent_aggregate(state_path: Path, sid: str, role: str) -> dict[str, Any] | None: + """返回活跃 turn 下当前 agent 的聚合摘要。""" + turn = get_current_turn(state_path, sid) + if not isinstance(turn, dict): + return None + agent_spans = turn.get("agent_spans") + if not isinstance(agent_spans, dict): + return None + rec = agent_spans.get(_normalize_agent(role)) + if not isinstance(rec, dict): + return None + aggregate = rec.get("aggregate") + if not isinstance(aggregate, dict): + return None + return json.loads(json.dumps(aggregate, ensure_ascii=False)) + + +def _step_span_key(agent: str, message_id: str) -> str: + """为 step span 生成按 agent 与 message_id 唯一定位的 key。""" + return f"{agent}|{message_id}" + + +def upsert_step_span( + state_path: Path, + sid: str, + agent: str, + message_id: str, + carrier: dict[str, str], + *, + transcript_path: str | None = None, +) -> dict[str, str]: + """在当前 turn 下登记一个 step span carrier，并对 `(turn, agent, message_id)` 保持幂等。""" + now = time.time() + normalized_agent = str(agent or "main").strip() or "main" + normalized_message_id = str(message_id or "").strip() + if not normalized_message_id: + return {} + + def _update(state: dict[str, Any]) -> dict[str, str]: + payload = get_agentlens_session(state, sid) + if payload.get("version") != AGENTLENS_SCHEMA_VERSION: + migrated = _migrate_agentlens_v1_to_v2(dict(payload), state.get(sid) if isinstance(state.get(sid), dict) else None) + payload.clear() + payload.update(migrated) + current = payload.get("current_turn") + if not isinstance(current, dict): + return {} + steps = current.setdefault("step_spans", {}) + if not isinstance(steps, dict): + steps = {} + current["step_spans"] = steps + key = _step_span_key(normalized_agent, normalized_message_id) + existing = steps.get(key) + if isinstance(existing, dict) and isinstance(existing.get("carrier"), dict) and existing["carrier"]: + return dict(existing["carrier"]) + steps[key] = { + "carrier": dict(carrier or {}), + "opened_at": now, + "agent": normalized_agent, + "message_id": normalized_message_id, + "transcript_path": transcript_path or "", + } + payload["version"] = AGENTLENS_SCHEMA_VERSION + return dict(carrier or {}) + + return update_state_locked(state_path, _update) + + +def upsert_step_span_atomic( + state_path: Path, + sid: str, + agent: str, + message_id: str, + *, + carrier_factory: Any, + transcript_path: str | None = None, +) -> dict[str, str]: + """在写锁内原子地获取或创建一个 step span carrier。 + + `carrier_factory` 是一个无参可调用对象，用来生成 carrier 字典。 + 只有在 span 还不存在时才会被调用，从而避免这样的 TOCTOU 竞争： + ``_generate_step_carrier`` 已经先产生了不可逆的 zhiyanllm span， + 但随后另一个 hook 进程已经写入同一 step，导致本次创建失去归属。 + """ + now = time.time() + normalized_agent = str(agent or "main").strip() or "main" + normalized_message_id = str(message_id or "").strip() + if not normalized_message_id: + return {} + + def _update(state: dict[str, Any]) -> dict[str, str]: + payload = get_agentlens_session(state, sid) + if payload.get("version") != AGENTLENS_SCHEMA_VERSION: + migrated = _migrate_agentlens_v1_to_v2(dict(payload), state.get(sid) if isinstance(state.get(sid), dict) else None) + payload.clear() + payload.update(migrated) + current = payload.get("current_turn") + if not isinstance(current, dict): + return {} + steps = current.setdefault("step_spans", {}) + if not isinstance(steps, dict): + steps = {} + current["step_spans"] = steps + key = _step_span_key(normalized_agent, normalized_message_id) + existing = steps.get(key) + if isinstance(existing, dict) and isinstance(existing.get("carrier"), dict) and existing["carrier"]: + return dict(existing["carrier"]) + new_carrier = carrier_factory() + if not isinstance(new_carrier, dict) or not new_carrier.get("traceparent"): + return {} + steps[key] = { + "carrier": dict(new_carrier), + "opened_at": now, + "agent": normalized_agent, + "message_id": normalized_message_id, + "transcript_path": transcript_path or "", + } + payload["version"] = AGENTLENS_SCHEMA_VERSION + return dict(new_carrier) + + return update_state_locked(state_path, _update) + + +def get_step_span_carrier(state_path: Path, sid: str, agent: str, message_id: str) -> dict[str, str] | None: + """返回活跃 turn 下已持久化的 step span carrier；如果没有则返回 None。""" + turn = get_current_turn(state_path, sid) + if not isinstance(turn, dict): + return None + steps = turn.get("step_spans") + if not isinstance(steps, dict): + return None + key = _step_span_key(str(agent or "main").strip() or "main", str(message_id or "").strip()) + rec = steps.get(key) + if not isinstance(rec, dict): + return None + carrier = rec.get("carrier") + return dict(carrier) if isinstance(carrier, dict) and carrier else None + + +def get_latest_session_id(state: dict[str, Any]) -> str | None: + """返回 state 中最近启动的 session id。""" + latest_sid: str | None = None + latest_started_at = -1.0 + for sid, sess in state.items(): + if sid.startswith("_") or not isinstance(sess, dict): + continue + started_at = float(sess.get("started_at", 0) or 0) + if started_at >= latest_started_at: + latest_started_at = started_at + latest_sid = sid + return latest_sid + + +def get_pending_tool_emits(state_path: Path, sid: str) -> list[dict[str, Any]]: + """返回等待与 transcript/message_id 关联的缓冲 tool 事件。""" + with state_read_lock(state_path): + state = load_state(state_path) + sess = state.get(sid) + if not isinstance(sess, dict): + return [] + pending = sess.get("_pending_tool_emits") + if not isinstance(pending, list): + return [] + return [dict(item) for item in pending if isinstance(item, dict)] + + +def append_pending_tool_emit(state_path: Path, sid: str, tool_event: dict[str, Any]) -> list[dict[str, Any]]: + """缓存一条 tool 事件，延后再做 message_id 关联。""" + + def _update(state: dict[str, Any]) -> list[dict[str, Any]]: + sess = ensure_session(state, sid) + pending = sess.setdefault("_pending_tool_emits", []) + if not isinstance(pending, list): + pending = [] + sess["_pending_tool_emits"] = pending + pending.append(dict(tool_event)) + return [dict(item) for item in pending if isinstance(item, dict)] + + return update_state_locked(state_path, _update) + + +def replace_pending_tool_emits(state_path: Path, sid: str, tool_events: list[dict[str, Any]]) -> list[dict[str, Any]]: + """覆盖某个 session 的缓冲 tool 事件队列。""" + + def _update(state: dict[str, Any]) -> list[dict[str, Any]]: + sess = ensure_session(state, sid) + normalized = [dict(item) for item in tool_events if isinstance(item, dict)] + sess["_pending_tool_emits"] = normalized + return [dict(item) for item in normalized] + + return update_state_locked(state_path, _update) + +def bump_skill( + sess: dict, + name: str, + *, + via: str, + tool: str | None, + meta: dict | None = None, + submodule: str | None = None, + agent: str | None = None, +) -> None: + """增加 skill 使用计数，并按需记录子模块与 agent 维度。""" + if not name: + return + now = time.time() + rec = sess["skills"].setdefault(name, { + "count": 0, + "first_ts": now, + "last_ts": now, + "tools": [], + "via": [], + "source": None, + "version": None, + "submodule_hits": {}, + "by_agent": {}, + }) + rec["count"] += 1 + rec["last_ts"] = now + if tool and tool not in rec["tools"]: + rec["tools"].append(tool) + if via not in rec.get("via", []): + rec.setdefault("via", []).append(via) + if meta: + if meta.get("source") and not rec.get("source"): + rec["source"] = meta["source"] + if meta.get("version") and not rec.get("version"): + rec["version"] = meta["version"] + # 子模块命中统计 + if submodule: + sub_dict = rec.setdefault("submodule_hits", {}) + if not isinstance(sub_dict, dict): + sub_dict = {} + rec["submodule_hits"] = sub_dict + sh = sub_dict.setdefault(submodule, { + "count": 0, "first_ts": now, "last_ts": now, "tools": [], + }) + sh["count"] += 1 + sh["last_ts"] = now + if tool and tool not in sh["tools"]: + sh["tools"].append(tool) + # 按 agent 维度统计 + if agent: + ag_dict = rec.setdefault("by_agent", {}) + if not isinstance(ag_dict, dict): + ag_dict = {} + rec["by_agent"] = ag_dict + ar = ag_dict.setdefault(agent, { + "count": 0, "first_ts": now, "last_ts": now, + "tools": [], "submodules": [], + }) + ar["count"] += 1 + ar["last_ts"] = now + if tool and tool not in ar["tools"]: + ar["tools"].append(tool) + if submodule and submodule not in ar["submodules"]: + ar["submodules"].append(submodule) + + +def bump_rule( + sess: dict, + name: str, + *, + via: str, + tool: str | None, + meta: dict | None = None, + agent: str | None = None, +) -> None: + """增加 rule 使用计数。""" + if not name: + return + now = time.time() + rec = sess["rules"].setdefault(name, { + "count": 0, + "first_ts": now, + "last_ts": now, + "tools": [], + "via": [], + "source": None, + "by_agent": {}, + }) + rec["count"] += 1 + rec["last_ts"] = now + if tool and tool not in rec["tools"]: + rec["tools"].append(tool) + if via not in rec.get("via", []): + rec.setdefault("via", []).append(via) + if meta and meta.get("source") and not rec.get("source"): + rec["source"] = meta["source"] + if agent: + ag = rec.setdefault("by_agent", {}) + if not isinstance(ag, dict): + ag = {} + rec["by_agent"] = ag + ar = ag.setdefault(agent, {"count": 0, "last_ts": now}) + ar["count"] += 1 + ar["last_ts"] = now + + +def prune_sessions(state: dict, max_sessions: int = 5) -> None: + """在 state 中只保留最近的 N 个 session。""" + # 先过滤掉非 session 键（以下划线开头） + session_keys = [k for k in state if not k.startswith("_")] + if len(session_keys) > max_sessions: + to_remove = sorted( + session_keys, + key=lambda s: state[s].get("started_at", 0) if isinstance(state[s], dict) else 0, + )[:len(session_keys) - max_sessions] + for k in to_remove: + del state[k] + + +def _transcript_scope_key(transcript_path: str | None = None) -> str: + """把 transcript 路径归一化成内部 scope key。""" + path = str(transcript_path or "").strip() + return path or "__session__" + + +def _ensure_transcript_scopes(sess: dict) -> dict[str, dict[str, Any]]: + """确保 session 下存在 transcript scope 容器，并兼容旧字段迁移。""" + scopes = sess.setdefault("_transcripts", {}) + if not isinstance(scopes, dict): + scopes = {} + sess["_transcripts"] = scopes + + legacy_scope = scopes.get("__session__") + if not isinstance(legacy_scope, dict): + legacy_scope = {} + scopes["__session__"] = legacy_scope + + if "offset" not in legacy_scope and "_transcript_offset" in sess: + legacy_scope["offset"] = int(sess.get("_transcript_offset", 0) or 0) + + legacy_usage = sess.get("_last_cumulative_usage") + if "last_cumulative_usage" not in legacy_scope and isinstance(legacy_usage, dict) and legacy_usage: + legacy_scope["last_cumulative_usage"] = legacy_usage + + legacy_dedup = sess.get("_last_usage_event") + if "last_usage_event" not in legacy_scope and isinstance(legacy_dedup, dict) and legacy_dedup: + legacy_scope["last_usage_event"] = legacy_dedup + + return scopes + + +def _get_transcript_scope( + sess: dict, + transcript_path: str | None = None, + *, + create: bool = False, +) -> dict[str, Any] | None: + """获取指定 transcript 的 scope；必要时按需创建。""" + scopes = _ensure_transcript_scopes(sess) + key = _transcript_scope_key(transcript_path) + scope = scopes.get(key) + if isinstance(scope, dict): + return scope + if create: + scope = {} + scopes[key] = scope + return scope + return None + + +def get_transcript_offset(state: dict, sid: str, transcript_path: str | None = None) -> int: + """获取某个 session/transcript 最近一次上报的 transcript 文件 offset。""" + sess = state.get(sid) + if not isinstance(sess, dict): + return 0 + + scope = _get_transcript_scope(sess, transcript_path) + if isinstance(scope, dict): + return int(scope.get("offset", 0) or 0) + + if _transcript_scope_key(transcript_path) == "__session__": + return int(sess.get("_transcript_offset", 0) or 0) + return 0 + + +def set_transcript_offset( + state: dict, + sid: str, + offset: int, + transcript_path: str | None = None, +) -> None: + """在成功上报后持久化 transcript 文件 offset。""" + sess = state.get(sid) + if not isinstance(sess, dict): + return + + scope = _get_transcript_scope(sess, transcript_path, create=True) + if isinstance(scope, dict): + scope["offset"] = int(offset or 0) + + if _transcript_scope_key(transcript_path) == "__session__": + sess["_transcript_offset"] = int(offset or 0) + + +def get_last_cumulative_usage( + state: dict, + sid: str, + transcript_path: str | None = None, +) -> dict[str, int]: + """获取某个 transcript 最近一次上报的累计 token usage。""" + sess = state.get(sid) + if not isinstance(sess, dict): + return {} + + scope = _get_transcript_scope(sess, transcript_path) + if isinstance(scope, dict): + v = scope.get("last_cumulative_usage") + if isinstance(v, dict): + return v + + if _transcript_scope_key(transcript_path) == "__session__": + v = sess.get("_last_cumulative_usage") + if isinstance(v, dict): + return v + return {} + + +def get_last_llm_offset( + state: dict, + sid: str, + transcript_path: str | None = None, +) -> int: + """获取最近一次处理到的 LLM source_offset，用于增量构建 I/O。""" + sess = state.get(sid) + if not isinstance(sess, dict): + return 0 + scope = _get_transcript_scope(sess, transcript_path) + if isinstance(scope, dict): + v = scope.get("last_llm_offset") + if isinstance(v, (int, float)): + return int(v) + return 0 + + +def set_last_llm_offset( + state: dict, + sid: str, + offset: int, + transcript_path: str | None = None, +) -> None: + """持久化某个 transcript 最近处理到的 LLM source_offset。""" + sess = state.get(sid) + if not isinstance(sess, dict): + return + scope = _get_transcript_scope(sess, transcript_path, create=True) + if isinstance(scope, dict): + scope["last_llm_offset"] = int(offset or 0) + + +def set_last_cumulative_usage( + state: dict, + sid: str, + usage: dict[str, int], + transcript_path: str | None = None, +) -> None: + """在处理完一个 turn 后持久化累计 token usage。""" + sess = state.get(sid) + if not isinstance(sess, dict): + return + + scope = _get_transcript_scope(sess, transcript_path, create=True) + if isinstance(scope, dict): + scope["last_cumulative_usage"] = usage + + if _transcript_scope_key(transcript_path) == "__session__": + sess["_last_cumulative_usage"] = usage + + +def get_last_usage_event( + state: dict, + sid: str, + transcript_path: str | None = None, +) -> dict[str, Any] | None: + """获取某个 transcript 最近一次用于去重的 usage 快照。""" + sess = state.get(sid) + if not isinstance(sess, dict): + return None + + scope = _get_transcript_scope(sess, transcript_path) + if isinstance(scope, dict): + event = scope.get("last_usage_event") + if isinstance(event, dict): + return event + + if _transcript_scope_key(transcript_path) == "__session__": + event = sess.get("_last_usage_event") + if isinstance(event, dict): + return event + return None + + +def set_last_usage_event( + state: dict, + sid: str, + usage_event: dict[str, Any], + transcript_path: str | None = None, +) -> None: + """持久化某个 transcript 最近一次用于去重的 usage 快照。""" + sess = state.get(sid) + if not isinstance(sess, dict): + return + + scope = _get_transcript_scope(sess, transcript_path, create=True) + if isinstance(scope, dict): + scope["last_usage_event"] = usage_event + + if _transcript_scope_key(transcript_path) == "__session__": + sess["_last_usage_event"] = usage_event + + +def claim_transcript_event( + state_path: Path, + sid: str, + event_key: str, + transcript_path: str | None = None, + *, + max_seen_keys: int = 512, +) -> bool: + """跨进程原子地声明一个 transcript 作用域内的事件键。""" + seen_path = get_transcript_seen_path(state_path) + scope_key = f"{sid}|{_transcript_scope_key(transcript_path)}" + + with state_lock(state_path): + if seen_path.exists(): + try: + ledger = json.loads(seen_path.read_text("utf-8")) + except Exception: + ledger = {} + else: + ledger = {} + if not isinstance(ledger, dict): + ledger = {} + + seen = ledger.setdefault(scope_key, {}) + if not isinstance(seen, dict): + seen = {} + ledger[scope_key] = seen + if event_key in seen: + return False + seen[event_key] = time.time() + if len(seen) > max_seen_keys: + stale_keys = sorted(seen, key=lambda k: float(seen.get(k, 0) or 0))[:-max_seen_keys] + for key in stale_keys: + seen.pop(key, None) + seen_path.write_text(json.dumps(ledger, ensure_ascii=False), encoding="utf-8") + return True + + +def commit_transcript_progress( + state_path: Path, + sid: str, + transcript_path: str | None = None, + *, + offset: int | None = None, + last_cumulative_usage: dict[str, int] | None = None, +) -> None: + """在处理完一个窗口后，原子地持久化 transcript 解析进度。""" + + def _update(state: dict[str, Any]) -> None: + sess = ensure_session(state, sid) + scope = _get_transcript_scope(sess, transcript_path, create=True) + if not isinstance(scope, dict): + return + if offset is not None: + current_offset = int(scope.get("offset", 0) or 0) + next_offset = int(offset or 0) + if next_offset > current_offset: + scope["offset"] = next_offset + if _transcript_scope_key(transcript_path) == "__session__": + sess["_transcript_offset"] = next_offset + if isinstance(last_cumulative_usage, dict) and last_cumulative_usage: + scope["last_cumulative_usage"] = last_cumulative_usage + if _transcript_scope_key(transcript_path) == "__session__": + sess["_last_cumulative_usage"] = last_cumulative_usage + + update_state_locked(state_path, _update) diff --git a/.claude/skills/agent-observability/scripts/dashboard/index.html b/.claude/skills/agent-observability/scripts/dashboard/index.html new file mode 100644 index 0000000..8c391f5 --- /dev/null +++ b/.claude/skills/agent-observability/scripts/dashboard/index.html @@ -0,0 +1,1086 @@ + + + + + +Agent Observability + + + + + +
+
+
+
Agent Observability
+
读取本地 dashboard-data.json（由 build_dashboard_data.py 从 metrics.ndjson / workflow-state.json 聚合而成）· LoopForge 项目
+
+
加载中…
+
+ +
+ 时间范围 + + + +
+ +
+ +
+ + + + + +
+ +
+
+
建议关注按当前范围内的阈值规则实时算出，不是固定文案
+
+
+ +
+
+ 每日成本 +
cost_usd（按 config/pricing.json 估算）
+
+
+
+ +
+
+
+ Token 构成 +
+
+
+
+
+
按模型的成本占比Top 4 + 其他
+
+
+
+ +
+
数据表图表的完整数值备份
+
日期会话数成本 (USD)inputoutputcache_read工具失败
+
+
+ + + + + + + + + +
+ 数据来源：python3 ../build_dashboard_data.py --project-root <devflow 项目根目录> 会读取该项目下 + .codebuddy/skills/agent-observability/logs/metrics.ndjson、.state.json、 + artifacts/*/workflow-state.json（Classic/Portable 两种 schema 自动识别）与 + .codebuddy/hooks/logs/auto-dispatch.log，聚合成同目录下的 dashboard-data.json。 + 这个页面只读那份快照，不直接碰任何日志/状态文件——重新生成快照后刷新页面即可看到最新数据。 + 全部处理在本机完成，不发往任何外部服务。 +
+
+ + + + diff --git a/.claude/skills/agent-observability/scripts/main.py b/.claude/skills/agent-observability/scripts/main.py new file mode 100644 index 0000000..4ca2481 --- /dev/null +++ b/.claude/skills/agent-observability/scripts/main.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +"""agent-observability 的 hook 入口。""" +from __future__ import annotations + +import sys +from pathlib import Path + + +if __package__ in (None, ""): + _SCRIPTS_DIR = Path(__file__).resolve().parent + if str(_SCRIPTS_DIR) not in sys.path: + sys.path.insert(0, str(_SCRIPTS_DIR)) + from core.runtime import main as runtime_main # type: ignore +else: + from .core.runtime import main as runtime_main + + +def main() -> int: + return runtime_main(sys.argv[1:]) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.claude/skills/agent-observability/scripts/run_hook.sh b/.claude/skills/agent-observability/scripts/run_hook.sh new file mode 100644 index 0000000..d476810 --- /dev/null +++ b/.claude/skills/agent-observability/scripts/run_hook.sh @@ -0,0 +1,16 @@ +#!/bin/sh +set -eu + +PROJECT_DIR="${CODEBUDDY_PROJECT_DIR:-$(cd "$(dirname "$0")"/../../../.. && pwd -P)}" +PYTHON_BIN="$PROJECT_DIR/.venv/bin/python" +MAIN_PY="$PROJECT_DIR/.codebuddy/skills/agent-observability/scripts/main.py" + +# 只有当 .venv 里可用 zhiyanllm 时才优先使用该解释器； +# 否则回退到已安装 zhiyanllm 的系统 python3。 +if [ -x "$PYTHON_BIN" ]; then + if "$PYTHON_BIN" -c "import zhiyanllm" 2>/dev/null; then + exec "$PYTHON_BIN" "$MAIN_PY" "$@" + fi +fi + +exec python3 "$MAIN_PY" "$@" diff --git a/.claude/skills/agent-observability/templates/settings-hook.json b/.claude/skills/agent-observability/templates/settings-hook.json new file mode 100644 index 0000000..6da0900 --- /dev/null +++ b/.claude/skills/agent-observability/templates/settings-hook.json @@ -0,0 +1,62 @@ +{ + "_comment": "将本文件合并到 ~/.claude/settings.json 或 /.claude/settings.json", + "hooks": { + "SessionStart": [ + { + "hooks": [ + { + "type": "command", + "command": "\"$CODEBUDDY_PROJECT_DIR\"/.claude/skills/agent-observability/scripts/run_hook.sh session-start", + "timeout": 5 + } + ] + } + ], + "UserPromptSubmit": [ + { + "hooks": [ + { + "type": "command", + "command": "\"$CODEBUDDY_PROJECT_DIR\"/.claude/skills/agent-observability/scripts/run_hook.sh user-prompt-submit", + "timeout": 5 + } + ] + } + ], + "PreToolUse": [ + { + "matcher": "", + "hooks": [ + { + "type": "command", + "command": "\"$CODEBUDDY_PROJECT_DIR\"/.claude/skills/agent-observability/scripts/run_hook.sh pre", + "timeout": 5 + } + ] + } + ], + "PostToolUse": [ + { + "matcher": "", + "hooks": [ + { + "type": "command", + "command": "\"$CODEBUDDY_PROJECT_DIR\"/.claude/skills/agent-observability/scripts/run_hook.sh post", + "timeout": 30 + } + ] + } + ], + "Stop": [ + { + "hooks": [ + { + "type": "command", + "command": "\"$CODEBUDDY_PROJECT_DIR\"/.claude/skills/agent-observability/scripts/run_hook.sh stop", + "timeout": 30 + } + ] + } + ] + } +} diff --git a/.claude/skills/agent-observability/tests/fixtures/transcript.jsonl b/.claude/skills/agent-observability/tests/fixtures/transcript.jsonl new file mode 100644 index 0000000..a5ea47d --- /dev/null +++ b/.claude/skills/agent-observability/tests/fixtures/transcript.jsonl @@ -0,0 +1,10 @@ +{"type":"system","subtype":"init","session_id":"demo-001","content":"\ndo not exec\n\n\npdf\nhermes-devflow\n"} +{"type":"assistant","providerData":{"messageId":"msg-001"},"content":[{"type":"tool_use","name":"read_file","input":{"filePath":"README.md"}}],"usage":{"input_tokens":1820,"output_tokens":42,"cache_read_input_tokens":15300}} +{"type":"function_call","name":"Read","callId":"call-001","providerData":{"messageId":"msg-001"}} +{"type":"function_call_result","name":"Read","callId":"call-001","providerData":{"messageId":"msg-001"}} +{"type":"assistant","providerData":{"messageId":"msg-002"},"content":[{"type":"tool_use","name":"use_skill","input":{"command":"pdf"}}],"usage":{"input_tokens":2100,"output_tokens":58,"cache_read_input_tokens":15300}} +{"type":"function_call","name":"Bash","callId":"call-002","providerData":{"messageId":"msg-002"}} +{"type":"function_call","name":"Bash","callId":"call-003","providerData":{"messageId":"msg-002"}} +{"type":"function_call_result","name":"Bash","callId":"call-002","providerData":{"messageId":"msg-002"}} +{"type":"function_call_result","name":"Bash","callId":"call-003","providerData":{"messageId":"msg-002"}} +{"type":"assistant","providerData":{"messageId":"msg-003"},"content":"完成","usage":{"input_tokens":2240,"output_tokens":120,"cache_read_input_tokens":15300}} diff --git a/.claude/skills/agent-observability/tests/test_agent_identity.py b/.claude/skills/agent-observability/tests/test_agent_identity.py new file mode 100644 index 0000000..664cc1d --- /dev/null +++ b/.claude/skills/agent-observability/tests/test_agent_identity.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path +import sys + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "scripts" +if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) + +from core.agent_identity import AgentIdentityResolver # type: ignore +from core import state as st # type: ignore + + +class AgentIdentityResolverTests(unittest.TestCase): + def test_extract_identity_from_send_message_json(self): + resolver = AgentIdentityResolver() + data = { + "tool_name": "send_message", + "tool_input": { + "content": '{"from_role":"developer","next_target":{"role_name":"leader"}}' + }, + } + current, dispatched = resolver.extract_identity(data) + self.assertEqual(current, "developer") + self.assertEqual(dispatched, "leader") + + def test_track_returns_active_and_dispatched(self): + resolver = AgentIdentityResolver() + with tempfile.TemporaryDirectory() as td: + state_path = Path(td) / ".state.json" + sid = "s2" + state_data = st.load_state(state_path) + st.ensure_session(state_data, sid) + st.save_state(state_path, state_data) + + data = { + "tool_name": "Task", + "tool_input": {"subagent_name": "developer"}, + } + active, dispatched = resolver.track(state_path=state_path, sid=sid, data=data) + self.assertEqual(active, "main") + self.assertEqual(dispatched, "developer") + + state2 = st.load_state(state_path) + sess = state2.get(sid, {}) + self.assertEqual(sess.get("current_agent"), "developer") + self.assertEqual(sess.get("dispatched", {}).get("developer"), 1) + + def test_track_recognizes_agent_tool_subagent_type(self): + """Claude Code 的 Agent 工具用 subagent_type（不是 CodeBuddy 原生 Task 的 + subagent_name）标识派发目标；这条路径之前没有测试覆盖，_extract_from_task + 漏认这个字段会导致 dispatch 统计对所有 Agent 工具派发的场景失明。""" + resolver = AgentIdentityResolver() + with tempfile.TemporaryDirectory() as td: + state_path = Path(td) / ".state.json" + sid = "s3" + state_data = st.load_state(state_path) + st.ensure_session(state_data, sid) + st.save_state(state_path, state_data) + + data = { + "tool_name": "Agent", + "tool_input": {"subagent_type": "worker", "description": "spawn a team"}, + } + active, dispatched = resolver.track(state_path=state_path, sid=sid, data=data) + self.assertEqual(active, "main") + self.assertEqual(dispatched, "worker") + + state2 = st.load_state(state_path) + sess = state2.get(sid, {}) + self.assertEqual(sess.get("dispatched", {}).get("worker"), 1) + hist = sess.get("agent_history") or [] + self.assertTrue(any( + h.get("agent") == "worker" and str(h.get("evidence") or "").startswith("dispatch") + for h in hist + )) + + def test_extract_from_task_prefers_subagent_type_over_subagent_name(self): + resolver = AgentIdentityResolver() + result = resolver._extract_from_task({"subagent_type": "explorer", "subagent_name": "developer"}) + self.assertEqual(result, "explorer") diff --git a/.claude/skills/agent-observability/tests/test_build_dashboard_data.py b/.claude/skills/agent-observability/tests/test_build_dashboard_data.py new file mode 100644 index 0000000..53b031c --- /dev/null +++ b/.claude/skills/agent-observability/tests/test_build_dashboard_data.py @@ -0,0 +1,603 @@ +from __future__ import annotations + +import contextlib +import io +import json +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "scripts" +if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) + +import build_dashboard_data as bdd # type: ignore + + +class ToolCallFailedTests(unittest.TestCase): + """真实 CodeBuddy CLI transcript 里从没出现过 `is_error` 这个键——实测抓到的 + rawResponse 形如 {"exitCode": 0, "tool_error_code": "0", ...}（成功）或非 0 + exitCode（失败）。之前只认 is_error，导致这个宿主上失败统计永远是 0。""" + + def test_real_success_shape_from_codebuddy_is_not_a_failure(self): + # 实测数据：ls 一个不存在的路径，但命令写成了 `ls ...; echo "EXIT_CODE=$?"`， + # 复合命令整体 exitCode 被内层 echo 冲成了 0——工具调用本身没有失败。 + raw_response = { + "exitCode": 0, "signal": None, "interrupted": False, + "sandboxDenied": False, "stderrBytesTruncated": 0, + "stdoutBytesTruncated": 0, "tool_error_code": "0", + } + self.assertFalse(bdd.tool_call_failed(raw_response)) + + def test_nonzero_exit_code_is_a_failure(self): + raw_response = {"exitCode": 1, "tool_error_code": "1"} + self.assertTrue(bdd.tool_call_failed(raw_response)) + + def test_nonzero_tool_error_code_without_exit_code_is_a_failure(self): + raw_response = {"tool_error_code": "127"} + self.assertTrue(bdd.tool_call_failed(raw_response)) + + def test_legacy_is_error_flag_still_recognized(self): + self.assertTrue(bdd.tool_call_failed({"is_error": True})) + + def test_missing_or_non_dict_raw_response_is_not_a_failure(self): + self.assertFalse(bdd.tool_call_failed(None)) + self.assertFalse(bdd.tool_call_failed("not a dict")) + self.assertFalse(bdd.tool_call_failed({})) + + def test_build_failures_and_session_status_use_exit_code(self): + events = [ + { + "event": "tool", "sid": "s1", "ts": 1.0, "tool": "Bash", + "tool_details": {"raw_response": {"exitCode": 1, "tool_error_code": "1"}}, + }, + ] + failures = bdd.build_failures(events) + self.assertEqual(len(failures), 1) + self.assertEqual(failures[0]["tool"], "Bash") + self.assertEqual(failures[0]["code"], "1") + + _, sessions = bdd.build_daily_and_sessions(events) + self.assertEqual(sessions[0]["status"], "error") + + +class ResolveInputPathsTests(unittest.TestCase): + def setUp(self): + self.skill_root = Path("/tmp/fake-skill-root") + + def test_defaults_fall_back_to_skill_root_logs(self): + metrics, state = bdd.resolve_input_paths(self.skill_root) + self.assertEqual(metrics, self.skill_root / "logs" / "metrics.ndjson") + self.assertEqual(state, self.skill_root / "logs" / ".state.json") + + def test_none_args_equivalent_to_empty_args(self): + m1, s1 = bdd.resolve_input_paths(self.skill_root, None, None) + m2, s2 = bdd.resolve_input_paths(self.skill_root, "", "") + self.assertEqual((m1, s1), (m2, s2)) + + def test_both_args_override(self): + metrics, state = bdd.resolve_input_paths( + self.skill_root, "/abs/a/metrics.ndjson", "/abs/b/.state.json" + ) + self.assertEqual(metrics, Path("/abs/a/metrics.ndjson").absolute()) + self.assertEqual(state, Path("/abs/b/.state.json").absolute()) + + def test_metrics_only_override_keeps_default_state(self): + metrics, state = bdd.resolve_input_paths( + self.skill_root, "/abs/a/metrics.ndjson", None + ) + self.assertEqual(metrics, Path("/abs/a/metrics.ndjson").absolute()) + self.assertEqual(state, self.skill_root / "logs" / ".state.json") + + def test_state_only_override_keeps_default_metrics(self): + metrics, state = bdd.resolve_input_paths( + self.skill_root, None, "/abs/b/.state.json" + ) + self.assertEqual(metrics, self.skill_root / "logs" / "metrics.ndjson") + self.assertEqual(state, Path("/abs/b/.state.json").absolute()) + + def test_tilde_expansion(self): + metrics, state = bdd.resolve_input_paths( + self.skill_root, "~/x/metrics.ndjson", "~/y/.state.json" + ) + self.assertEqual(metrics, (Path.home() / "x" / "metrics.ndjson").absolute()) + self.assertEqual(state, (Path.home() / "y" / ".state.json").absolute()) + + +class MainEndToEndTests(unittest.TestCase): + def _run_main(self, argv: list[str]) -> None: + old_argv = sys.argv + sys.argv = ["build_dashboard_data.py"] + argv + try: + self.assertEqual(bdd.main(), 0) + finally: + sys.argv = old_argv + + def test_default_paths_unchanged_and_reflected_in_source(self): + with tempfile.TemporaryDirectory() as td: + out = Path(td) / "dashboard-data.json" + self._run_main(["--project-root", td, "--out", str(out)]) + self.assertTrue(out.is_file()) + data = json.loads(out.read_text("utf-8")) + expected_metrics = str(bdd.SCRIPTS_DIR.parent / "logs" / "metrics.ndjson") + expected_state = str(bdd.SCRIPTS_DIR.parent / "logs" / ".state.json") + self.assertEqual(data["source"]["metrics_ndjson"], expected_metrics) + self.assertEqual(data["source"]["state_json"], expected_state) + + def test_overridden_paths_read_and_reflected_in_source(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + metrics_file = root / "metrics.ndjson" + state_file = root / ".state.json" + metrics_file.write_text( + '{"event":"usage","sid":"s1","ts":1,"tokens":{"input":10,"output":20}}\n', + encoding="utf-8", + ) + state_file.write_text(json.dumps({"s1": {"skills": {}}}), encoding="utf-8") + out = root / "dashboard-data.json" + + self._run_main([ + "--project-root", td, + "--out", str(out), + "--metrics-path", str(metrics_file), + "--state-path", str(state_file), + ]) + self.assertTrue(out.is_file()) + + data = json.loads(out.read_text("utf-8")) + # resolve_input_paths 对覆盖路径执行 .expanduser().absolute()，断言需对齐（macOS 下不用 .resolve() 以避免 /tmp → /private/tmp 符号链接错位）。 + self.assertEqual(data["source"]["metrics_ndjson"], str(metrics_file.absolute())) + self.assertEqual(data["source"]["state_json"], str(state_file.absolute())) + # 覆盖文件确实被读取：metrics 行被解析进了 event_count。 + self.assertEqual(data["source"]["event_count"], 1) + + +class TopSlowTests(unittest.TestCase): + """`--top-slow N` 是纯增量能力：默认不开启，开启后也只往终端打印，绝不改动 + dashboard-data.json 的结构（看板前端按字段取值，多一个字段或少一个字段都会 + 直接影响渲染）。这些用例把"降序取前 N""非 tool 事件排除""ms 兜底""只读" + 四条边界钉住，避免后续有人顺手把结果写进 JSON 或改了排序方向。""" + + def _events(self): + return [ + {"event": "user_prompt_submit", "sid": "s1", "ts": 1.0, "turn_id": "t1", "agent": "main"}, + {"event": "tool", "sid": "s1", "ts": 2.0, "tool": "Read", "ms": 300, "agent": "main"}, + {"event": "usage", "sid": "s1", "ts": 3.0, "tokens": {"input": 10, "output": 5}, "agent": "main"}, + {"event": "tool", "sid": "s1", "ts": 4.0, "tool": "Bash", "ms": 900, "agent": "main"}, + {"event": "tool", "sid": "s1", "ts": 5.0, "tool": "Grep", "ms": 120, "agent": "main"}, + ] + + def _write_metrics(self, root: Path) -> Path: + metrics = root / "metrics.ndjson" + metrics.write_text( + "\n".join(json.dumps(e) for e in self._events()) + "\n", + encoding="utf-8", + ) + return metrics + + def _run_main(self, argv: list[str]) -> str: + """跑一次 main() 并返回捕获到的 stdout——top-slow 的产出只体现在终端上， + 不体现在 JSON 里，所以断言必须落在 stdout。""" + old_argv = sys.argv + sys.argv = ["build_dashboard_data.py"] + argv + buf = io.StringIO() + try: + with contextlib.redirect_stdout(buf): + rc = bdd.main() + finally: + sys.argv = old_argv + self.assertEqual(rc, 0) + return buf.getvalue() + + def _printed_tools(self, stdout: str) -> list[str]: + tools = [] + for line in stdout.splitlines(): + line = line.strip() + if not line or not line[0].isdigit(): + continue + tools.append(line.split(". ", 1)[1].split()[0]) + return tools + + def test_returns_n_slowest_tool_events_descending(self): + rows = bdd.build_top_slow(self._events(), 2) + self.assertEqual([(r["tool"], r["ms"]) for r in rows], [("Bash", 900), ("Read", 300)]) + + def test_limit_greater_than_event_count_returns_all_without_error(self): + events = [{"event": "tool", "sid": "s1", "ts": 1.0, "tool": "Bash", "ms": 10}] + rows = bdd.build_top_slow(events, 99) + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["tool"], "Bash") + + def test_non_tool_events_are_excluded(self): + rows = bdd.build_top_slow(self._events(), 10) + self.assertEqual([r["tool"] for r in rows], ["Bash", "Read", "Grep"]) + # usage 事件即使带 ms 也不能混进来。 + self.assertTrue(all(isinstance(r["ms"], (int, float)) for r in rows)) + + def test_missing_or_non_numeric_ms_falls_back_to_zero(self): + events = [ + {"event": "tool", "sid": "s1", "ts": 1.0, "tool": "Bash"}, + {"event": "tool", "sid": "s1", "ts": 2.0, "tool": "Read", "ms": "120"}, + {"event": "tool", "sid": "s1", "ts": 3.0, "tool": "Grep", "ms": None}, + {"event": "tool", "sid": "s1", "ts": 4.0, "tool": "Edit", "ms": 1}, + ] + rows = bdd.build_top_slow(events, 4) + self.assertEqual(len(rows), 4) + # 非数字/缺失的 ms 一律按 0 计，不会把字符串 "120" 排到最前面。 + self.assertEqual(sum(r["ms"] for r in rows), 1) + + def test_disabled_by_default_prints_nothing_extra(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + metrics = self._write_metrics(root) + out = root / "dashboard-data.json" + stdout = self._run_main([ + "--project-root", td, "--out", str(out), + "--metrics-path", str(metrics), "--state-path", str(root / ".state.json"), + ]) + self.assertIn("wrote ", stdout) + self.assertNotIn("top-slow", stdout) + self.assertEqual(self._printed_tools(stdout), []) + + def test_enabled_prints_tool_and_ms_descending(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + metrics = self._write_metrics(root) + out = root / "dashboard-data.json" + stdout = self._run_main([ + "--project-root", td, "--out", str(out), + "--metrics-path", str(metrics), "--state-path", str(root / ".state.json"), + "--top-slow", "2", + ]) + self.assertIn("wrote ", stdout) + self.assertIn("top-slow", stdout) + self.assertEqual(self._printed_tools(stdout), ["Bash", "Read"]) + self.assertIn("900ms", stdout) + self.assertNotIn("120ms", stdout) # N=2，第三名的 Grep 不该出现 + + def test_enabled_does_not_change_dashboard_data_structure(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + metrics = self._write_metrics(root) + state = root / ".state.json" + state.write_text(json.dumps({"s1": {"skills": {}}}), encoding="utf-8") + # 输出到两个不同文件再比对，避免"第二次跑覆盖了第一次"导致对比失真。 + out_on = root / "on.json" + out_off = root / "off.json" + self._run_main([ + "--project-root", td, "--out", str(out_on), + "--metrics-path", str(metrics), "--state-path", str(state), "--top-slow", "5", + ]) + self._run_main([ + "--project-root", td, "--out", str(out_off), + "--metrics-path", str(metrics), "--state-path", str(state), + ]) + data_on = json.loads(out_on.read_text("utf-8")) + data_off = json.loads(out_off.read_text("utf-8")) + data_on.pop("generated_at") + data_off.pop("generated_at") + self.assertEqual(data_on, data_off) + # 兜底断言：结果没有以任何形式渗进 JSON（不只是"结构相同"）。 + self.assertNotIn("topSlow", json.dumps(data_on)) + self.assertNotIn("top-slow", json.dumps(data_on)) + + +class BuildDailyAndSessionsTurnAttributionTests(unittest.TestCase): + """真实场景复现过：AgentLens（写 state.current_turn 的那套 tracing）关闭时， + tool/usage 事件自己的 turn_id 永远是 None，只有 user_prompt_submit 事件带真实 + turn_id。修复前所有没有 turn_id 的事件会被塌缩进同一个 "?" 占位桶，看起来 + 像"整个会话只有 1 个 turn"，哪怕实际发了好几轮 prompt。""" + + def _events(self): + return [ + {"event": "user_prompt_submit", "sid": "s1", "ts": 1.0, "turn_id": "t1", "agent": "main"}, + {"event": "tool", "sid": "s1", "ts": 2.0, "tool": "Bash", "ms": 100, "agent": "main"}, + {"event": "usage", "sid": "s1", "ts": 3.0, "tokens": {"input": 10, "output": 5}, "agent": "main"}, + {"event": "user_prompt_submit", "sid": "s1", "ts": 4.0, "turn_id": "t2", "agent": "main"}, + {"event": "tool", "sid": "s1", "ts": 5.0, "tool": "Read", "ms": 50, "agent": "main"}, + {"event": "tool", "sid": "s1", "ts": 6.0, "tool": "Edit", "ms": 80, "agent": "main"}, + ] + + def test_turns_reflect_prompt_boundaries_not_a_single_bucket(self): + daily, sessions = bdd.build_daily_and_sessions(self._events()) + self.assertEqual(len(sessions), 1) + sess = sessions[0] + self.assertEqual(sess["turns"], 2) + self.assertEqual(sess["toolCalls"], 3) + self.assertEqual(len(sess["timeline"]), 2) + turn1_tools = [e["tool"] for e in sess["timeline"][0]["events"] if e["kind"] == "tool"] + turn2_tools = [e["tool"] for e in sess["timeline"][1]["events"] if e["kind"] == "tool"] + self.assertEqual(turn1_tools, ["Bash"]) + self.assertEqual(turn2_tools, ["Read", "Edit"]) + + def test_events_before_first_prompt_fall_back_to_unknown_bucket(self): + events = [ + {"event": "tool", "sid": "s1", "ts": 1.0, "tool": "Bash", "ms": 100, "agent": "main"}, + {"event": "user_prompt_submit", "sid": "s1", "ts": 2.0, "turn_id": "t1", "agent": "main"}, + {"event": "tool", "sid": "s1", "ts": 3.0, "tool": "Read", "ms": 50, "agent": "main"}, + ] + _, sessions = bdd.build_daily_and_sessions(events) + sess = sessions[0] + # 没有归属到任何真实 turn 的事件仍然单独成桶，不会被错误地并入第一个真实 turn。 + self.assertEqual(len(sess["timeline"]), 2) + self.assertEqual(sess["timeline"][0]["turn"], 1) + + def test_prompt_only_turn_with_no_tool_calls_still_appears_in_timeline(self): + """真实数据复现过：纯对话轮次（用户发了 prompt，但助手没有调用任何工具， + 也没有产生 usage 事件）之前完全不会在 timeline 里建桶——turns 计数是对的 + （来自 user_prompt_submit 自带的 turn_id），但展开的时间线分组数会比 turns + 少，二者对不上。""" + events = [ + {"event": "user_prompt_submit", "sid": "s1", "ts": 1.0, "turn_id": "t1", "agent": "main"}, + {"event": "tool", "sid": "s1", "ts": 2.0, "tool": "Read", "ms": 100, "agent": "main"}, + {"event": "user_prompt_submit", "sid": "s1", "ts": 3.0, "turn_id": "t2", "agent": "main"}, + {"event": "user_prompt_submit", "sid": "s1", "ts": 4.0, "turn_id": "t3", "agent": "main"}, + {"event": "tool", "sid": "s1", "ts": 5.0, "tool": "Grep", "ms": 50, "agent": "main"}, + ] + _, sessions = bdd.build_daily_and_sessions(events) + sess = sessions[0] + self.assertEqual(sess["turns"], 3) + self.assertEqual(len(sess["timeline"]), 3) + self.assertEqual(sess["timeline"][1]["events"], []) + + def test_prompt_only_session_still_counted_in_daily_session_count(self): + """真实数据复现过（TC5 当天 5 个 session，"总览"页汇总出的会话数却是 4）： + daily["_sids"]（"会话浏览"总览卡片"会话数"的数据源）之前只在 tool/usage + 分支里 add，一个全程只发了 prompt、没有触发任何工具调用也没有 usage 事件 + 的 session（比如用户发了消息但被拒绝/打断，没有真正执行）会被这天的 + sessionCount 完全漏掉——尽管 sessions 列表（"会话浏览"tab）里它确实在， + 导致总览页"会话数"比实际能展开看到的会话数少。""" + events = [ + {"event": "user_prompt_submit", "sid": "s1", "ts": 1.0, "turn_id": "t1", "agent": "main"}, + {"event": "tool", "sid": "s1", "ts": 2.0, "tool": "Read", "ms": 50, "agent": "main"}, + {"event": "user_prompt_submit", "sid": "s2", "ts": 3.0, "turn_id": "t1", "agent": "main"}, + ] + daily, sessions = bdd.build_daily_and_sessions(events) + self.assertEqual(len(sessions), 2) + self.assertEqual(len(daily), 1) + self.assertEqual(daily[0]["sessionCount"], 2) + + +class BuildDevflowRunsClassicSoloTests(unittest.TestCase): + """真实运行验证过：Classic small 任务会经过 PHASE-0 -> SOLO -> TASK-05 + （knowledge 由 solo-developer 合并执行），不是只有 PHASE-0/SOLO 两步。""" + + def test_solo_run_includes_task05_and_marks_phase0_completed(self): + with tempfile.TemporaryDirectory() as td: + project_root = Path(td) + run_dir = project_root / "artifacts" / "solo-with-knowledge_20260914_0600" + run_dir.mkdir(parents=True) + (run_dir / "workflow-state.json").write_text(json.dumps({ + "version": "1.3", + "task_slug": "solo-with-knowledge_20260914_0600", + "size_class": "small", + "current_stage": "SUMMARY", + "last_event": "workflow_completed", + # 注意：真实 schema 里 stages{} 不含 PHASE-0 —— Phase 0 是隐式完成的。 + "stages": { + "SOLO": {"status": "completed", "executor": "solo-developer", "retry_count": 0}, + "TASK-02": {"status": "skipped", "executor": "architect", "retry_count": 0}, + "TASK-03": {"status": "skipped", "executor": "developer", "retry_count": 0}, + "CODE-REVIEW": {"status": "skipped", "executor": "code-reviewer", "retry_count": 0}, + "TASK-04": {"status": "skipped", "executor": "test-engineer", "retry_count": 0}, + "TASK-05": {"status": "completed", "executor": "solo-developer", "retry_count": 0}, + }, + }), encoding="utf-8") + + runs = bdd.build_devflow_runs(project_root) + self.assertEqual(len(runs), 1) + stage_by_key = {s["key"]: s["status"] for s in runs[0]["stages"]} + self.assertIn("TASK-05", stage_by_key) + self.assertEqual(stage_by_key["TASK-05"], "completed") + self.assertEqual(stage_by_key["PHASE-0"], "completed") + self.assertNotIn("TASK-01", stage_by_key) # SOLO 路径不该混进 medium/large 的阶段 + + +class BuildDevflowRunsMediumWithLingeringSoloKeyTests(unittest.TestCase): + """用户反馈复现：workflow-state.json 模板固定给每次运行都写一份 SOLO stage + （见 assets/workflow-state-template.json 里 SOLO.description 的说明： + "仅当 size_class==small 时启用，否则保持 skipped"），medium/large 任务从不会 + 真正执行它，状态停在 "pending" 或 "skipped"。修复前 `"SOLO" in stages` 只看 + key 存不存在，不看状态，导致这类 medium 任务被误判成 solo 路径，阶段视图会用 + CLASSIC_SOLO_ORDER 渲染，把 TASK-01/TASK-02/TASK-03/CODE-REVIEW/TASK-04 全部 + 从 stepper 里漏掉。""" + + def _run(self, solo_status: str): + with tempfile.TemporaryDirectory() as td: + project_root = Path(td) + run_dir = project_root / "artifacts" / "medium-with-lingering-solo_20260915_0900" + run_dir.mkdir(parents=True) + (run_dir / "workflow-state.json").write_text(json.dumps({ + "version": "1.3", + "task_slug": "medium-with-lingering-solo_20260915_0900", + "size_class": "medium", + "current_stage": "TASK-03", + "last_event": "TASK-02_completed", + "stages": { + "SOLO": {"status": solo_status, "executor": "solo-developer", "retry_count": 0}, + "TASK-01": {"status": "completed", "executor": "main", "retry_count": 0}, + "TASK-02": {"status": "completed", "executor": "architect", "retry_count": 0}, + "TASK-03": {"status": "in_progress", "executor": "developer", "retry_count": 0}, + "CODE-REVIEW": {"status": "pending", "executor": "code-reviewer", "retry_count": 0}, + "TASK-04": {"status": "pending", "executor": "test-engineer", "retry_count": 0}, + "TASK-05": {"status": "pending", "executor": "knowledge-engineer", "retry_count": 0}, + }, + }), encoding="utf-8") + return bdd.build_devflow_runs(project_root) + + def test_medium_run_with_skipped_solo_key_uses_full_classic_order(self): + runs = self._run("skipped") + self.assertEqual(len(runs), 1) + stage_by_key = {s["key"]: s["status"] for s in runs[0]["stages"]} + # 修复前：这里会走 CLASSIC_SOLO_ORDER，TASK-01/02/03/CODE-REVIEW/TASK-04 全部消失。 + self.assertIn("TASK-01", stage_by_key) + self.assertIn("TASK-02", stage_by_key) + self.assertIn("TASK-03", stage_by_key) + self.assertIn("CODE-REVIEW", stage_by_key) + self.assertIn("TASK-04", stage_by_key) + self.assertEqual(stage_by_key["TASK-02"], "completed") + self.assertEqual(stage_by_key["TASK-03"], "in_progress") + + def test_medium_run_with_pending_solo_key_uses_full_classic_order(self): + # 模板初始状态是 "pending"（不是所有实现都会显式改成 "skipped"），同样不该被判成 solo。 + runs = self._run("pending") + stage_by_key = {s["key"]: s["status"] for s in runs[0]["stages"]} + self.assertIn("TASK-01", stage_by_key) + self.assertIn("CODE-REVIEW", stage_by_key) + + +class BuildDevflowRunsDurationTests(unittest.TestCase): + """真实数据复现过：TASK-05 的 started_at 可能早于 SOLO 的 completed_at + （同一次 solo-developer 执行内部的子步骤，不是真正先后发生的两个阶段）。 + total_duration 必须按真实起止跨度算，不能对逐阶段 duration 求和——否则会把 + 重叠部分重复计入，虚高于源数据本身反映的运行时长（这次修复前是 600s， + 真实跨度只有 540s）。""" + + def _write_state(self, project_root: Path, stages: dict) -> None: + run_dir = project_root / "artifacts" / "overlap-run_20260914_0620" + run_dir.mkdir(parents=True) + (run_dir / "workflow-state.json").write_text(json.dumps({ + "version": "1.3", + "task_slug": "overlap-run_20260914_0620", + "size_class": "small", + "current_stage": "SUMMARY", + "last_event": "workflow_completed", + "stages": stages, + }), encoding="utf-8") + + def test_total_duration_uses_true_span_not_sum_of_overlapping_stages(self): + with tempfile.TemporaryDirectory() as td: + project_root = Path(td) + self._write_state(project_root, { + "SOLO": { + "status": "completed", "executor": "solo-developer", "retry_count": 0, + "started_at": "2026-09-14T06:20:00Z", "completed_at": "2026-09-14T06:29:00Z", + }, + "TASK-05": { + "status": "completed", "executor": "solo-developer", "retry_count": 0, + # 早于 SOLO 的 completed_at —— 真实数据里观测到的重叠场景。 + "started_at": "2026-09-14T06:28:00Z", "completed_at": "2026-09-14T06:29:00Z", + }, + }) + runs = bdd.build_devflow_runs(project_root) + self.assertEqual(len(runs), 1) + # 真实跨度是 06:20~06:29 = 540s，不是 540+60=600s。 + self.assertEqual(runs[0]["duration"], 540) + stage_by_key = {s["key"]: s["duration"] for s in runs[0]["stages"]} + # 单个阶段自身的 duration 不受影响，仍然如实反映各自的起止跨度。 + self.assertEqual(stage_by_key["SOLO"], 540) + self.assertEqual(stage_by_key["TASK-05"], 60) + + def test_total_duration_is_zero_when_no_stage_has_valid_timestamps(self): + with tempfile.TemporaryDirectory() as td: + project_root = Path(td) + self._write_state(project_root, { + "SOLO": {"status": "completed", "executor": "solo-developer", "retry_count": 0}, + "TASK-05": {"status": "completed", "executor": "solo-developer", "retry_count": 0}, + }) + runs = bdd.build_devflow_runs(project_root) + self.assertEqual(runs[0]["duration"], 0) + + +class BuildDailyAndSessionsPerEventAgentTests(unittest.TestCase): + """真实数据复现过（TC5，size_class=small 但走了真实 team_create/send_message + 派发）：同一个 sid 下，metrics.ndjson 的 tool/usage 事件本来就各自带着准确的 + agent 字段（一次真实会话里 66 条 main、64 条 solo-developer），但展开进 + timeline 的每条 event 字典只有 kind/tool/ms/err，从不写回 agent —— 时间线里 + 完全看不出某次工具调用到底是 main 自己做的还是派发给 solo-developer 后做的， + 等于白白丢弃了上游已经采集到的数据。""" + + def _events(self): + return [ + {"event": "user_prompt_submit", "sid": "s1", "ts": 1.0, "turn_id": "t1", "agent": "main"}, + {"event": "tool", "sid": "s1", "ts": 2.0, "tool": "Agent", "ms": 50, "agent": "main"}, + {"event": "tool", "sid": "s1", "ts": 3.0, "tool": "Bash", "ms": 100, "agent": "solo-developer"}, + {"event": "usage", "sid": "s1", "ts": 4.0, "tokens": {"input": 10, "output": 5}, "agent": "solo-developer"}, + {"event": "tool", "sid": "s1", "ts": 5.0, "tool": "SendMessage", "ms": 30, "agent": "main"}, + ] + + def test_timeline_events_carry_the_agent_that_actually_ran_them(self): + _, sessions = bdd.build_daily_and_sessions(self._events()) + sess = sessions[0] + events = sess["timeline"][0]["events"] + by_tool = {e.get("tool"): e for e in events if e["kind"] == "tool"} + self.assertEqual(by_tool["Agent"]["agent"], "main") + self.assertEqual(by_tool["Bash"]["agent"], "solo-developer") + self.assertEqual(by_tool["SendMessage"]["agent"], "main") + usage_events = [e for e in events if e["kind"] == "usage"] + self.assertEqual(usage_events[0]["agent"], "solo-developer") + + def test_missing_agent_on_raw_event_falls_back_to_main(self): + events = [ + {"event": "user_prompt_submit", "sid": "s1", "ts": 1.0, "turn_id": "t1"}, + {"event": "tool", "sid": "s1", "ts": 2.0, "tool": "Read", "ms": 10}, + ] + _, sessions = bdd.build_daily_and_sessions(events) + sess = sessions[0] + self.assertEqual(sess["timeline"][0]["events"][0]["agent"], "main") + + +class BuildDispatchTests(unittest.TestCase): + """真实数据复现过（TC5 的 dashboard-data-top-slow 会话，sid 01a09ef5...）： + 修复前 build_dispatch() 把 agent_history 里所有 evidence 以 "dispatch" 或 + "inbox" 开头的条目一律计数，导致两个真实问题—— + 1) solo-developer 被上报回声（inbox@report:*）重复计入，真实派发只有 2 次， + 却显示成 5； + 2) team-lead 显示成一个"被 main 派发的子 agent"，但 team-lead 在 + core/agent_identity.py::normalize_role_name 里本来就和 main 归为一类 + （它是 CodeBuddy 原生 team 基础设施里 lead session 自己的 mailbox 名， + 不是真实存在的子 agent），面板标题明明叫"Main → 子 Agent 派发次数"， + 混进一个其实等价于 main 自己的假子 agent。""" + + def _real_tc5_agent_history(self): + # 逐条取自真实会话 01a09ef5-1c6b-7928-870e-a5056ff360e1 的 .state.json。 + return [ + {"ts": 1.0, "agent": "solo-developer", "evidence": "dispatch@Agent<-main"}, + {"ts": 2.0, "agent": "team-lead", "evidence": "dispatch@SendMessage<-solo-developer"}, + {"ts": 3.0, "agent": "solo-developer", "evidence": "inbox@report:solo-developer"}, + {"ts": 4.0, "agent": "solo-developer", "evidence": "inbox@dispatch:solo-developer"}, + {"ts": 5.0, "agent": "main", "evidence": "dispatch@SendMessage<-solo-developer"}, + {"ts": 6.0, "agent": "main", "evidence": "inbox@handoff:solo-developer->main"}, + {"ts": 7.0, "agent": "solo-developer", "evidence": "dispatch@SendMessage<-main"}, + {"ts": 8.0, "agent": "solo-developer", "evidence": "inbox@report:solo-developer"}, + ] + + def test_real_tc5_history_excludes_team_lead_and_report_echoes(self): + state = {"01a09ef5": {"agent_history": self._real_tc5_agent_history()}} + rows = bdd.build_dispatch(state) + # team-lead（main 的别名）完全不出现；solo-developer 只数真正代表新派发的 + # 3 条证据（2 条工具触发的 dispatch@ + 1 条 inbox 独立确认的 dispatch:）， + # 2 条上报回声（inbox@report:*）不计入。 + self.assertEqual(rows, [{"agent": "solo-developer", "count": 3}]) + + def test_inbox_dispatch_evidence_alone_is_still_counted(self): + """inbox 扫描独立确认的 "main 派给了谁"（inbox@dispatch:*）是原生 team + 基础设施里唯一能感知到、但没有经过 Agent/Task 工具调用拦截到的派发方式 + （例如 team-lead 用 mailbox 直接投递任务），必须保留，不能因为过滤 + report/handoff 回声就连这类真实派发信号也一起丢掉。""" + state = {"s1": {"agent_history": [ + {"ts": 1.0, "agent": "architect", "evidence": "inbox@dispatch:architect"}, + ]}} + rows = bdd.build_dispatch(state) + self.assertEqual(rows, [{"agent": "architect", "count": 1}]) + + def test_report_and_handoff_evidence_alone_are_not_dispatches(self): + state = {"s1": {"agent_history": [ + {"ts": 1.0, "agent": "developer", "evidence": "inbox@report:developer"}, + {"ts": 2.0, "agent": "test-engineer", "evidence": "inbox@handoff:developer->test-engineer"}, + ]}} + rows = bdd.build_dispatch(state) + self.assertEqual(rows, []) + + def test_team_lead_is_excluded_even_without_report_noise(self): + state = {"s1": {"agent_history": [ + {"ts": 1.0, "agent": "team-lead", "evidence": "dispatch@SendMessage<-main"}, + ]}} + rows = bdd.build_dispatch(state) + self.assertEqual(rows, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/.claude/skills/agent-observability/tests/test_cls_sink.py b/.claude/skills/agent-observability/tests/test_cls_sink.py new file mode 100644 index 0000000..2da9d6a --- /dev/null +++ b/.claude/skills/agent-observability/tests/test_cls_sink.py @@ -0,0 +1,184 @@ +from __future__ import annotations + +import json +import os +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "scripts" +if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) + +from core import cls_sink, emitter # type: ignore + + +class CLSSinkTests(unittest.TestCase): + def _write_cls_env(self, root: Path, *extra_lines: str) -> None: + lines = [ + "CLS_TOPIC_ID=test-topic-id", + "CLS_ENDPOINT=cls.internal.tencentcloudapi.com", + "CLS_SERVICE_NAME=agent-observability", + "CLS_UPLOAD_TIMEOUT_SECONDS=20", + *extra_lines, + ] + (root / ".env").write_text("\n".join(lines) + "\n", encoding="utf-8") + + def test_load_config_supports_cls_and_tc_credentials(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + self._write_cls_env( + root, + "TC_SECRET_ID=test-secret-id", + "TC_SECRET_KEY=test-secret-key", + ) + with mock.patch.dict(os.environ, {}, clear=True): + config = cls_sink.load_config(td) + + self.assertTrue(config.enabled) + self.assertEqual(config.endpoint, "cls.internal.tencentcloudapi.com") + self.assertEqual(config.topic_id, "test-topic-id") + self.assertEqual(config.secret_id, "test-secret-id") + self.assertEqual(config.secret_key, "test-secret-key") + self.assertEqual(config.secret_token, "") + self.assertEqual(config.service_name, "agent-observability") + self.assertEqual(config.timeout_seconds, 20) + self.assertTrue(config.ready) + + def test_load_config_supports_tencentcloud_scoped_credentials(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + self._write_cls_env(root) + with mock.patch.dict( + os.environ, + { + "TENCENTCLOUD_SECRET_ID_438167613": "scoped-secret-id", + "TENCENTCLOUD_SECRET_KEY_438167613": "scoped-secret-key", + }, + clear=True, + ): + config = cls_sink.load_config(td) + + self.assertEqual(config.secret_id, "scoped-secret-id") + self.assertEqual(config.secret_key, "scoped-secret-key") + self.assertEqual(config.timeout_seconds, 20) + self.assertTrue(config.ready) + + def test_mirror_record_invokes_node_uploader(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + self._write_cls_env( + root, + "CLS_SECRET_ID=test-secret-id", + "CLS_SECRET_KEY=test-secret-key", + ) + with mock.patch.object(cls_sink, "_put_logs_via_api", return_value=(False, "python-failed")), \ + mock.patch.object(cls_sink.subprocess, "run", return_value=mock.Mock(returncode=0, stdout="", stderr="")) as run, \ + mock.patch.dict(os.environ, {}, clear=True): + ok = cls_sink.mirror_record({"event": "tool", "sid": "s-1"}, cwd=td) + + self.assertTrue(ok) + run.assert_called_once() + payload = json.loads(run.call_args.kwargs["input"]) + self.assertEqual(payload["records"][0]["event"], "tool") + self.assertEqual(payload["topicId"], "test-topic-id") + self.assertEqual(payload["secretToken"], "") + + def test_load_config_requires_credentials_from_env(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + self._write_cls_env(root) + with mock.patch.dict(os.environ, {}, clear=True): + config = cls_sink.load_config(td) + + self.assertEqual(config.secret_id, "") + self.assertEqual(config.secret_key, "") + self.assertEqual(config.secret_token, "") + self.assertEqual(config.timeout_seconds, 20) + self.assertFalse(config.ready) + + def test_load_config_reads_secret_token_aliases(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + self._write_cls_env( + root, + "CLS_SECRET_ID=test-secret-id", + "CLS_SECRET_KEY=test-secret-key", + "TC_SESSION_TOKEN=test-session-token", + ) + with mock.patch.dict(os.environ, {}, clear=True): + config = cls_sink.load_config(td) + + self.assertEqual(config.secret_token, "test-session-token") + + def test_load_config_supports_timeout_override(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + self._write_cls_env( + root, + "CLS_SECRET_ID=test-secret-id", + "CLS_SECRET_KEY=test-secret-key", + "CLS_UPLOAD_TIMEOUT_SECONDS=45", + ) + with mock.patch.dict(os.environ, {}, clear=True): + config = cls_sink.load_config(td) + + self.assertEqual(config.timeout_seconds, 45) + + def test_mirror_record_writes_debug_when_uploader_fails(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + debug_path = root / "cls-push-debug.ndjson" + self._write_cls_env( + root, + "CLS_SECRET_ID=test-secret-id", + "CLS_SECRET_KEY=test-secret-key", + ) + with mock.patch.object(cls_sink, "debug_log_path", return_value=debug_path), \ + mock.patch.object(cls_sink, "_put_logs_via_api", return_value=(False, "python-failed")), \ + mock.patch.object(cls_sink.subprocess, "run", return_value=mock.Mock(returncode=1, stdout="", stderr="boom")), \ + mock.patch.dict(os.environ, {}, clear=True): + ok = cls_sink.mirror_record({"event": "tool", "sid": "s-2"}, cwd=td) + + entries = [json.loads(line) for line in debug_path.read_text("utf-8").splitlines() if line.strip()] + + self.assertFalse(ok) + self.assertEqual(entries[-1]["stage"], "uploader_failed") + self.assertEqual(entries[-1]["stderr"], "boom") + self.assertEqual(entries[-1]["timeout_seconds"], 20) + + def test_mirror_record_writes_success_debug_when_uploader_succeeds(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + debug_path = root / "cls-push-debug.ndjson" + self._write_cls_env( + root, + "CLS_SECRET_ID=test-secret-id", + "CLS_SECRET_KEY=test-secret-key", + ) + with mock.patch.object(cls_sink, "debug_log_path", return_value=debug_path), \ + mock.patch.object(cls_sink, "_put_logs_via_api", return_value=(True, "ok")), \ + mock.patch.dict(os.environ, {}, clear=True): + ok = cls_sink.mirror_record({"event": "tool", "sid": "s-3"}, cwd=td) + + entries = [json.loads(line) for line in debug_path.read_text("utf-8").splitlines() if line.strip()] + + self.assertTrue(ok) + self.assertEqual(entries[0]["stage"], "uploader_start") + self.assertEqual(entries[-1]["stage"], "uploader_ok") + self.assertEqual(entries[-1]["method"], "python_api_v3") + + def test_emit_keeps_local_log_when_cls_push_raises(self): + with tempfile.TemporaryDirectory() as td: + log_path = Path(td) / "metrics.ndjson" + with mock.patch.object(emitter.cls_sink, "mirror_record", side_effect=RuntimeError("boom")): + emitter.emit(log_path, {"event": "start", "sid": "s-emit"}) + + records = [json.loads(line) for line in log_path.read_text("utf-8").splitlines() if line.strip()] + + self.assertEqual(len(records), 1) + self.assertEqual(records[0]["event"], "start") + self.assertEqual(records[0]["sid"], "s-emit") diff --git a/.claude/skills/agent-observability/tests/test_collector.py b/.claude/skills/agent-observability/tests/test_collector.py new file mode 100644 index 0000000..05547f0 --- /dev/null +++ b/.claude/skills/agent-observability/tests/test_collector.py @@ -0,0 +1,715 @@ +from __future__ import annotations + +import json +import tempfile +import time +import unittest +from pathlib import Path +import sys + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "scripts" +if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) + +from core import agent_identity, agentlens, collector, emitter, runtime, state as st # type: ignore + + +class CollectorTests(unittest.TestCase): + def test_cost_of_uses_known_model_price_table(self): + tokens = {"input": 1000, "output": 200, "cache_read": 0, "total": 1200} + cost = emitter.cost_of(tokens, "gpt-4o") + self.assertIsNotNone(cost) + self.assertGreaterEqual(cost, 0.0) + + def test_cost_of_returns_none_for_unknown_model(self): + tokens = {"input": 1000, "output": 200, "cache_read": 0, "total": 1200} + self.assertIsNone(emitter.cost_of(tokens, "unknown-model")) + + def test_emit_adds_codebuddy_cli_data_source(self): + with tempfile.TemporaryDirectory() as td: + log_path = Path(td) / "metrics.ndjson" + + emitter.emit(log_path, {"event": "start", "sid": "s-data-source"}) + + record = json.loads(log_path.read_text(encoding="utf-8").strip()) + self.assertEqual(record["data_source"], "codebuddy-cli") + + def test_emit_preserves_explicit_data_source(self): + with tempfile.TemporaryDirectory() as td: + log_path = Path(td) / "metrics.ndjson" + + emitter.emit(log_path, {"event": "start", "sid": "s-data-source", "data_source": "manual"}) + + record = json.loads(log_path.read_text(encoding="utf-8").strip()) + self.assertEqual(record["data_source"], "manual") + + def test_emit_session_duration_uses_first_log_ts_when_state_was_recreated(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + log_path = root / "metrics.ndjson" + state_path = root / ".state.json" + log_path.write_text( + json.dumps({"event": "start", "sid": "s-duration", "ts": 100.0}) + "\n", + encoding="utf-8", + ) + state_path.write_text( + json.dumps({"s-duration": {"started_at": 190.0}}, ensure_ascii=False), + encoding="utf-8", + ) + + emitter.emit(log_path, {"event": "usage", "sid": "s-duration", "ts": 200.0, "tokens": {"input": 1, "output": 2}}) + + records = [json.loads(line) for line in log_path.read_text(encoding="utf-8").splitlines()] + self.assertEqual(records[-1]["session_duration_sec"], 100.0) + + def test_emit_session_duration_starts_at_zero_without_prior_log_or_state(self): + with tempfile.TemporaryDirectory() as td: + log_path = Path(td) / "metrics.ndjson" + + emitter.emit(log_path, {"event": "start", "sid": "s-new", "ts": 100.0}) + + record = json.loads(log_path.read_text(encoding="utf-8").strip()) + self.assertEqual(record["session_duration_sec"], 0.0) + + def test_parse_transcript_tail_extracts_tokens(self): + transcript = ROOT / "tests" / "fixtures" / "transcript.jsonl" + out = collector.parse_transcript_tail(str(transcript), max_lines=20) + self.assertIsInstance(out, dict) + self.assertIsInstance(out.get("tokens"), dict) + self.assertEqual(out["tokens"].get("input_tokens"), 2240) + self.assertEqual(out["tokens"].get("output_tokens"), 120) + usage_records = out.get("usage_records") or [] + tool_records = out.get("tool_records") or [] + self.assertEqual(len(usage_records), 3) + self.assertEqual(usage_records[0]["tokens"].get("input_tokens"), 1820) + self.assertEqual(usage_records[-1]["tokens"].get("input_tokens"), 2240) + self.assertEqual(usage_records[0].get("message_id"), "msg-001") + self.assertEqual(usage_records[-1].get("message_id"), "msg-003") + self.assertLess(usage_records[0]["offset"], usage_records[-1]["offset"]) + self.assertEqual(len(tool_records), 8) + self.assertEqual(tool_records[0]["tool"], "read_file") + self.assertEqual(tool_records[0]["message_id"], "msg-001") + self.assertEqual(tool_records[0]["next_message_id"], "msg-002") + self.assertEqual(tool_records[-1]["tool"], "Bash") + self.assertEqual(tool_records[-1]["message_id"], "msg-002") + self.assertEqual(tool_records[-1]["next_message_id"], "msg-003") + + def test_realtime_usage_is_request_level_and_dedupes_across_phase(self): + transcript = ROOT / "tests" / "fixtures" / "transcript.jsonl" + with tempfile.TemporaryDirectory() as td: + state_path = Path(td) / ".state.json" + sid = "s-transcript" + + scan = collector.collect_transcript_entries( + state_path=state_path, + sid=sid, + transcript_path=str(transcript), + max_lines=20, + ) + entries = scan.get("entries") or [] + + self.assertEqual(len(entries), 3) + self.assertEqual(entries[0]["tokens"]["input"], 1820) + self.assertGreater(entries[0]["offset"], 0) + self.assertEqual(entries[0]["message_id"], "msg-001") + self.assertEqual(entries[1]["tokens"]["input"], 2100) + self.assertEqual(entries[1]["tokens"]["total"], 2158) + self.assertEqual(len(scan.get("tool_records") or []), 8) + + usage_key = collector.build_transcript_event_key( + kind="usage", + tool="Read", + entry=entries[0], + ) + stop_key = collector.build_transcript_event_key( + kind="stop", + tool="__stop__", + entry=entries[0], + ) + other_tool_key = collector.build_transcript_event_key( + kind="usage", + tool="Bash", + entry=entries[0], + ) + + self.assertEqual(usage_key, stop_key) + self.assertEqual(usage_key, other_tool_key) + self.assertTrue(st.claim_transcript_event(state_path, sid, usage_key, str(transcript))) + self.assertFalse(st.claim_transcript_event(state_path, sid, stop_key, str(transcript))) + + def test_find_current_tool_message_id_prefers_transcript_tool_records(self): + transcript = ROOT / "tests" / "fixtures" / "transcript.jsonl" + with tempfile.TemporaryDirectory() as td: + state_path = Path(td) / ".state.json" + sid = "s-tool-mid" + + read_mid = collector.find_current_tool_message_id( + state_path, + sid, + str(transcript), + "Read", + ) + bash_mid = collector.find_current_tool_message_id( + state_path, + sid, + str(transcript), + "Bash", + ) + + self.assertEqual(read_mid, "msg-002") + self.assertEqual(bash_mid, "msg-003") + + def test_resolve_subagent_transcript_alias_and_tool_context(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + sid = "session-123" + main = root / f"{sid}.jsonl" + bundle = root / sid + subagents = bundle / "subagents" + subagents.mkdir(parents=True) + alias_session_id = "subagent-session-1" + subagent = subagents / "agent-a.jsonl" + main.write_text("", encoding="utf-8") + subagent.write_text( + "\n".join( + [ + json.dumps( + { + "type": "message", + "sessionId": alias_session_id, + "providerData": {"agent": "Explore"}, + } + ), + json.dumps( + { + "type": "function_call", + "sessionId": alias_session_id, + "name": "Grep", + "callId": "call-1", + "providerData": { + "agent": "Explore", + "messageId": "msg-sub-001", + }, + } + ), + ] + ) + + "\n", + encoding="utf-8", + ) + + aliased_path = str(root / f"{alias_session_id}.jsonl") + resolved = collector.resolve_transcript_path_alias(sid, aliased_path) + self.assertEqual(resolved, str(subagent.resolve())) + + state_path = root / ".state.json" + context = collector.find_current_tool_context( + state_path=state_path, + sid=sid, + transcript_path=aliased_path, + tool_name="Grep", + ) + self.assertEqual( + context, + { + "message_id": "msg-sub-001", + "transcript_path": str(subagent.resolve()), + "agent": "explore", + "tool_details": {"call_id": "call-1"}, + }, + ) + + def test_find_current_tool_context_prefers_nearest_timestamp(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + transcript = root / "session.jsonl" + transcript.write_text( + "\n".join( + [ + json.dumps( + { + "timestamp": 1000, + "type": "function_call", + "name": "Bash", + "callId": "call-1", + "providerData": {"messageId": "msg-early"}, + } + ), + json.dumps( + { + "timestamp": 5000, + "type": "function_call", + "name": "Bash", + "callId": "call-2", + "providerData": {"messageId": "msg-late"}, + } + ), + ] + ) + + "\n", + encoding="utf-8", + ) + state_path = root / ".state.json" + + early = collector.find_current_tool_context( + state_path=state_path, + sid="s-nearest", + transcript_path=str(transcript), + tool_name="Bash", + event_ts=1.2, + ) + late = collector.find_current_tool_context( + state_path=state_path, + sid="s-nearest", + transcript_path=str(transcript), + tool_name="Bash", + event_ts=4.9, + ) + + self.assertEqual(early["message_id"], "msg-early") + self.assertEqual(late["message_id"], "msg-late") + + def test_find_current_tool_context_claims_distinct_call_ids_with_same_timestamp(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + transcript = root / "session.jsonl" + transcript.write_text( + "\n".join( + [ + json.dumps( + { + "timestamp": 5000, + "type": "function_call", + "name": "Bash", + "callId": "call-1", + "arguments": "{\"command\":\"echo one\"}", + "providerData": {"messageId": "msg-shared"}, + } + ), + json.dumps( + { + "timestamp": 5000, + "type": "function_call", + "name": "Bash", + "callId": "call-2", + "arguments": "{\"command\":\"echo two\"}", + "providerData": {"messageId": "msg-shared"}, + } + ), + json.dumps( + { + "timestamp": 5000, + "type": "function_call_result", + "name": "Bash", + "callId": "call-1", + "providerData": { + "messageId": "msg-shared", + "toolResult": {"content": "result one"}, + }, + } + ), + json.dumps( + { + "timestamp": 5000, + "type": "function_call_result", + "name": "Bash", + "callId": "call-2", + "providerData": { + "messageId": "msg-shared", + "toolResult": {"content": "result two"}, + }, + } + ), + ] + ) + + "\n", + encoding="utf-8", + ) + state_path = root / ".state.json" + + first = collector.find_current_tool_context( + state_path=state_path, + sid="s-claim-shared", + transcript_path=str(transcript), + tool_name="Bash", + event_ts=4.9, + claim=True, + ) + second = collector.find_current_tool_context( + state_path=state_path, + sid="s-claim-shared", + transcript_path=str(transcript), + tool_name="Bash", + event_ts=4.9, + claim=True, + ) + third = collector.find_current_tool_context( + state_path=state_path, + sid="s-claim-shared", + transcript_path=str(transcript), + tool_name="Bash", + event_ts=4.9, + claim=True, + ) + + self.assertEqual(first["message_id"], "msg-shared") + self.assertEqual(second["message_id"], "msg-shared") + self.assertNotEqual(first["tool_details"]["call_id"], second["tool_details"]["call_id"]) + self.assertIn("arguments", first["tool_details"]) + self.assertIn("result_content", first["tool_details"]) + self.assertEqual(third, {"duplicate": True}) + + def test_find_fallback_usage_event_prefers_latest_matching_usage(self): + tool_event = { + "tool": "Bash", + "agent": "main", + "transcript_path": "/tmp/demo.jsonl", + } + usage_events = [ + { + "agent": "main", + "tool": "Read", + "transcript_path": "/tmp/demo.jsonl", + "message_id": "msg-old", + "source_offset": 100, + }, + { + "agent": "main", + "tool": "Bash", + "transcript_path": "/tmp/demo.jsonl", + "message_id": "msg-current", + "source_offset": 200, + }, + ] + + matched = collector.find_fallback_usage_event(tool_event, usage_events) + self.assertIsNotNone(matched) + self.assertEqual(matched["message_id"], "msg-current") + + def test_find_current_tool_context_prefers_explicit_call_id_over_nearest_timestamp(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + transcript = root / "session.jsonl" + transcript.write_text( + "\n".join( + [ + json.dumps( + { + "timestamp": 1000, + "type": "function_call", + "name": "Bash", + "callId": "call-early", + "providerData": {"messageId": "msg-early"}, + } + ), + json.dumps( + { + "timestamp": 1001, + "type": "function_call_result", + "name": "Bash", + "callId": "call-early", + "providerData": { + "messageId": "msg-early", + "toolResult": {"content": "early result"}, + }, + } + ), + json.dumps( + { + "timestamp": 5000, + "type": "function_call", + "name": "Bash", + "callId": "call-late", + "providerData": {"messageId": "msg-late"}, + } + ), + json.dumps( + { + "timestamp": 5001, + "type": "function_call_result", + "name": "Bash", + "callId": "call-late", + "providerData": { + "messageId": "msg-late", + "toolResult": {"content": "late result"}, + }, + } + ), + ] + ) + + "\n", + encoding="utf-8", + ) + state_path = root / ".state.json" + + context = collector.find_current_tool_context( + state_path=state_path, + sid="s-call-id-priority", + transcript_path=str(transcript), + tool_name="Bash", + call_id="call-early", + event_ts=4.9, + ) + + self.assertEqual(context["message_id"], "msg-early") + self.assertEqual(context["tool_details"]["call_id"], "call-early") + self.assertEqual(context["tool_details"]["result_content"], "early result") + + def test_find_current_tool_context_uses_next_message_id_when_available(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + transcript = root / "session.jsonl" + transcript.write_text( + "\n".join( + [ + json.dumps( + { + "timestamp": 1000, + "type": "function_call", + "name": "Read", + "callId": "call-1", + "providerData": {"messageId": "msg-tool"}, + } + ), + json.dumps( + { + "timestamp": 1001, + "type": "function_call_result", + "name": "Read", + "callId": "call-1", + "providerData": { + "messageId": "msg-tool", + "toolResult": {"content": "missing"}, + }, + } + ), + json.dumps( + { + "timestamp": 2000, + "type": "assistant", + "providerData": {"messageId": "msg-next"}, + "content": "next step", + } + ), + ] + ) + + "\n", + encoding="utf-8", + ) + state_path = root / ".state.json" + + context = collector.find_current_tool_context( + state_path=state_path, + sid="s-next-mid", + transcript_path=str(transcript), + tool_name="Read", + call_id="call-1", + ) + + self.assertEqual(context["message_id"], "msg-next") + self.assertEqual(context["tool_details"]["original_message_id"], "msg-tool") + self.assertEqual(context["tool_details"]["next_message_id"], "msg-next") + self.assertTrue(context["tool_details"]["message_id_reassigned"]) + + def test_flush_pending_tool_events_emits_after_transcript_catches_up(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + state_path = root / ".state.json" + log_path = root / "metrics.ndjson" + transcript = root / "session.jsonl" + transcript.write_text("", encoding="utf-8") + sid = "s-pending" + + tool_event = { + "event": "tool", + "sid": sid, + "agent": "main", + "tool": "Bash", + "ms": 42, + "transcript_path": str(transcript), + "turn_id": "turn-1", + "ts": 1.2, + "skill": [], + "rule": [], + "cwd": str(ROOT), + } + st.append_pending_tool_emit(state_path, sid, tool_event) + + transcript.write_text( + json.dumps( + { + "timestamp": 1300, + "type": "function_call", + "name": "Bash", + "callId": "call-1", + "providerData": {"messageId": "msg-later"}, + } + ) + + "\n", + encoding="utf-8", + ) + + emitted_calls: list[dict[str, object]] = [] + original_emit_post_step = agentlens.emit_post_step + try: + agentlens.emit_post_step = lambda **kwargs: emitted_calls.append(kwargs) + runtime.flush_pending_tool_events( + state_path=state_path, + log_path=log_path, + sid=sid, + ) + finally: + agentlens.emit_post_step = original_emit_post_step + + self.assertEqual(st.get_pending_tool_emits(state_path, sid), []) + lines = [json.loads(line) for line in log_path.read_text(encoding="utf-8").splitlines() if line.strip()] + self.assertEqual(len(lines), 1) + self.assertEqual(lines[0]["message_id"], "msg-later") + self.assertEqual(len(emitted_calls), 1) + self.assertEqual(emitted_calls[0]["tool_event"]["message_id"], "msg-later") + + def test_related_transcript_paths_include_subagents(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + sid = "session-123" + main = root / f"{sid}.jsonl" + bundle = root / sid + subagents = bundle / "subagents" + subagents.mkdir(parents=True) + sub_a = subagents / "agent-a.jsonl" + sub_b = subagents / "agent-b.jsonl" + for path in (main, sub_a, sub_b): + path.write_text("", encoding="utf-8") + + paths = collector.related_transcript_paths(sid, str(main)) + + self.assertEqual( + paths, + [ + str(main.resolve()), + str(sub_a.resolve()), + str(sub_b.resolve()), + ], + ) + + def test_subagent_transcript_maps_to_role_name(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + subagent = root / "subagents" / "agent-27e41af0.jsonl" + subagent.parent.mkdir(parents=True) + subagent.write_text( + '{"content":[{"text":""}]}\n', + encoding="utf-8", + ) + + self.assertEqual( + agent_identity.agent_for_transcript_path(str(subagent), "main"), + "knowledge-engineer", + ) + + def test_subagent_transcript_prefers_assignment_role_over_general_provider_agent(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + subagent = root / "subagents" / "agent-165e361f.jsonl" + subagent.parent.mkdir(parents=True) + subagent.write_text( + '{"type":"message","providerData":{"agent":"general-purpose"},"content":[{"text":"\\n待命。\\n你在本 devflow 中的角色：developer（开发角色）"}]}\n', + encoding="utf-8", + ) + + self.assertEqual( + agent_identity.agent_for_transcript_path(str(subagent), "main"), + "developer", + ) + + def test_inbox_standby_assignment_does_not_switch_current_agent(self): + messages = [ + { + "mailbox_name": "architect", + "mailbox_role": "architect", + "from_role": "main", + "summary": "Initial task assignment for architect", + "text": "待命，监听 main 唤醒。", + "payload": None, + "is_shutdown": False, + "is_standby": True, + } + ] + current, dispatched, meta = agent_identity.infer_identity_from_messages(messages) + self.assertIsNone(current) + self.assertIsNone(dispatched) + self.assertEqual(meta["messages_seen"], 1) + + def test_record_pre_post_duration(self): + with tempfile.TemporaryDirectory() as td: + pending = Path(td) / ".pending.json" + pre = {"session_id": "s1", "tool_name": "read_file"} + post = {"session_id": "s1", "tool_name": "read_file"} + collector.record_pre(pending, pre) + time.sleep(0.02) + ms = collector.record_post(pending, post) + self.assertIsInstance(ms, int) + self.assertGreaterEqual(ms, 0) + + def test_record_tool_usage_counts_skill_and_rule(self): + with tempfile.TemporaryDirectory() as td: + state_path = Path(td) / ".state.json" + sid = "s-tool" + # 先确保 session 结构 + s = st.load_state(state_path) + st.ensure_session(s, sid) + st.save_state(state_path, s) + + data = { + "tool_name": "use_skill", + "tool_input": {"command": "pdf", "filePath": "README.md"}, + } + skills_meta = {"pdf": {"source": "user", "version": "1.0.0"}} + rules_meta = { + "security": {"source": "project", "alwaysApply": True, "enabled": True, "globs": []} + } + collector.record_tool_usage( + state_path=state_path, + sid=sid, + data=data, + skills_meta=skills_meta, + rules_meta=rules_meta, + active_agent="main", + collect_skills=True, + ) + + skills_usage, rules_usage = collector.get_session_usage(state_path, sid) + self.assertEqual(skills_usage.get("pdf", {}).get("count"), 1) + self.assertEqual(rules_usage.get("security", {}).get("count"), 1) + + def test_step_span_registry_is_idempotent(self): + with tempfile.TemporaryDirectory() as td: + state_path = Path(td) / ".state.json" + sid = "s-step" + carrier = {"traceparent": "00-" + ("1" * 32) + "-" + ("2" * 16) + "-01"} + st.begin_turn(state_path, sid, "turn-1", carrier) + + first = st.upsert_step_span( + state_path, + sid, + "main", + "msg-001", + {"traceparent": "00-" + ("1" * 32) + "-" + ("3" * 16) + "-01"}, + transcript_path="/tmp/demo.jsonl", + ) + second = st.upsert_step_span( + state_path, + sid, + "main", + "msg-001", + {"traceparent": "00-" + ("1" * 32) + "-" + ("4" * 16) + "-01"}, + transcript_path="/tmp/demo.jsonl", + ) + + self.assertEqual(first, second) + self.assertEqual(st.get_step_span_carrier(state_path, sid, "main", "msg-001"), first) + + +if __name__ == "__main__": + unittest.main() diff --git a/.claude/skills/agent-observability/tests/test_devflow.py b/.claude/skills/agent-observability/tests/test_devflow.py new file mode 100644 index 0000000..9ad0a7f --- /dev/null +++ b/.claude/skills/agent-observability/tests/test_devflow.py @@ -0,0 +1,389 @@ +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path +import sys + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "scripts" +if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) + +from core import agent_identity, devflow, state as st # type: ignore + + +def _write_team_config(team_dir: Path, *, lead_session_id: str, member_cwd: str) -> None: + team_dir.mkdir(parents=True, exist_ok=True) + (team_dir / "config.json").write_text( + json.dumps({ + "leadSessionId": lead_session_id, + "createdAt": 1, + "members": [{"cwd": member_cwd}], + }), + encoding="utf-8", + ) + + +class TaskSlugFromTeamDirTests(unittest.TestCase): + def test_strips_fixed_prefix(self): + team_dir = Path("/tmp/.codebuddy/teams/multi-agents-devflow-my-task_20260911_1200") + self.assertEqual(devflow.task_slug_from_team_dir(team_dir), "my-task_20260911_1200") + + def test_returns_none_for_non_devflow_team_dir(self): + team_dir = Path("/tmp/.codebuddy/teams/some-other-team") + self.assertIsNone(devflow.task_slug_from_team_dir(team_dir)) + + +class StageSnapshotTests(unittest.TestCase): + def test_projects_expected_fields_and_ignores_non_dict_stage(self): + workflow_state = { + "current_stage": "TASK-03", + "size_class": "medium", + "run_mode": "auto", + "stages": { + "TASK-02": {"status": "completed", "executor": "architect", "retry_count": 0, "review_result": None}, + "CODE-REVIEW": {"status": "failed", "executor": "code-reviewer", "retry_count": 1, "review_result": "failed"}, + "garbage": "not-a-dict", + }, + } + snap = devflow.stage_snapshot(workflow_state) + self.assertEqual(snap["current_stage"], "TASK-03") + self.assertEqual(snap["size_class"], "medium") + self.assertNotIn("garbage", snap["stages"]) + self.assertEqual(snap["stages"]["CODE-REVIEW"]["retry_count"], 1) + self.assertEqual(snap["stages"]["CODE-REVIEW"]["review_result"], "failed") + + def test_handles_missing_or_malformed_input(self): + self.assertEqual(devflow.stage_snapshot(None), {}) + self.assertEqual(devflow.stage_snapshot({"stages": "not-a-dict"}), { + "current_stage": None, "size_class": None, "run_mode": None, + "schema_version": None, "execution_mode": None, "host_adapter": None, "run_id": None, + "stages": {}, + }) + + +class DiffStageChangesTests(unittest.TestCase): + def test_first_observation_produces_no_changes(self): + curr = devflow.stage_snapshot({"stages": {"TASK-02": {"status": "completed", "retry_count": 0}}}) + self.assertEqual(devflow.diff_stage_changes(None, curr), []) + + def test_detects_retry_count_increase_as_a_change(self): + prev = devflow.stage_snapshot({"stages": {"CODE-REVIEW": {"status": "in_progress", "retry_count": 0}}}) + curr = devflow.stage_snapshot({"stages": {"CODE-REVIEW": {"status": "failed", "retry_count": 1, "review_result": "failed"}}}) + changes = devflow.diff_stage_changes(prev, curr) + self.assertEqual(len(changes), 1) + self.assertEqual(changes[0]["stage"], "CODE-REVIEW") + self.assertEqual(changes[0]["retry_count"], 1) + self.assertEqual(changes[0]["review_result"], "failed") + + def test_unchanged_stage_produces_no_entry(self): + snap = devflow.stage_snapshot({"stages": {"TASK-02": {"status": "completed", "retry_count": 0}}}) + self.assertEqual(devflow.diff_stage_changes(snap, snap), []) + + +class PtIdFromTranscriptPathTests(unittest.TestCase): + def test_extracts_pt_track_from_subagent_transcript_name(self): + path = "/tmp/session-123/subagents/sub-developer-PT-01.jsonl" + self.assertEqual(agent_identity.pt_id_from_transcript_path(path), "PT-01") + + def test_returns_none_for_non_subagent_or_non_pt_transcript(self): + self.assertIsNone(agent_identity.pt_id_from_transcript_path("/tmp/session-123.jsonl")) + self.assertIsNone(agent_identity.pt_id_from_transcript_path("/tmp/session-123/subagents/architect.jsonl")) + + +class ResolveAndDiffTests(unittest.TestCase): + def test_returns_none_when_no_devflow_team_found(self): + with tempfile.TemporaryDirectory() as td: + state_path = Path(td) / ".state.json" + result = devflow.resolve_and_diff(state_path, "sid-1", "/tmp/some-project") + self.assertIsNone(result) + state = st.load_state(state_path) + self.assertIsNone(state["sid-1"]["_devflow"]["context"]) + + def test_late_appearing_devflow_run_is_discovered_on_a_later_call(self): + """真实 bug 回归测试：同一个长生命周期 sid，会话开始时探测不到 devflow + （此时既没有 team 目录也没有 artifacts/），之后才真正跑起 devflow + （比如通过 Agent 工具后台 spawn，而不是从一开始就是独立 team 成员 sid）。 + 早期的"没找到"不能被永久缓存，必须在 artifacts/ 出现之后的下一次调用里发现它。""" + with tempfile.TemporaryDirectory() as td: + project_dir = Path(td) / "project" + project_dir.mkdir() + state_path = project_dir / ".codebuddy" / "skills" / "agent-observability" / "logs" / ".state.json" + sid = "sid-long-lived" + + # 会话早期：还没有任何 devflow 痕迹。 + first = devflow.resolve_and_diff(state_path, sid, str(project_dir)) + self.assertIsNone(first) + + # 同一个 sid，会话中途才出现 devflow 产物（没有 team 目录，走 artifacts 扫描兜底）。 + artifacts_dir = project_dir / "artifacts" / "late-appearing-task_20260914_1200" + artifacts_dir.mkdir(parents=True) + (artifacts_dir / "workflow-state.json").write_text(json.dumps({ + "version": "1.3", "current_stage": "SOLO", "size_class": "small", + "stages": {"SOLO": {"status": "completed", "executor": "solo-developer", "retry_count": 0}}, + }), encoding="utf-8") + + second = devflow.resolve_and_diff(state_path, sid, str(project_dir)) + self.assertIsNotNone(second) + self.assertEqual(second["task_slug"], "late-appearing-task_20260914_1200") + + def test_end_to_end_with_fake_team_and_workflow_state(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + project_dir = root / "project" + project_dir.mkdir() + config_home = root / "codebuddy-home" + teams_root = config_home / "teams" + team_dir = teams_root / "multi-agents-devflow-fix-token-bypass_20260911_0900" + _write_team_config(team_dir, lead_session_id="sid-42", member_cwd=str(project_dir)) + + artifacts_dir = project_dir / "artifacts" / "fix-token-bypass_20260911_0900" + artifacts_dir.mkdir(parents=True) + workflow_state_path = artifacts_dir / "workflow-state.json" + workflow_state_path.write_text(json.dumps({ + "current_stage": "TASK-03", + "size_class": "medium", + "stages": { + "TASK-02": {"status": "completed", "executor": "architect", "retry_count": 0}, + "TASK-03": {"status": "in_progress", "executor": "developer", "retry_count": 0}, + }, + }), encoding="utf-8") + + import os + old_env = os.environ.get("CODEBUDDY_CONFIG_DIR") + os.environ["CODEBUDDY_CONFIG_DIR"] = str(config_home) + try: + state_path = project_dir / ".codebuddy" / "skills" / "agent-observability" / "logs" / ".state.json" + sid = "sid-42" + + # 第一次调用：发现 devflow 上下文，但因为是首次观测不产出 changes。 + first = devflow.resolve_and_diff(state_path, sid, str(project_dir)) + self.assertIsNotNone(first) + self.assertEqual(first["task_slug"], "fix-token-bypass_20260911_0900") + self.assertEqual(first["current_stage"], "TASK-03") + self.assertEqual(first["changes"], []) + + # workflow-state.json 更新：TASK-03 打回重试。 + workflow_state_path.write_text(json.dumps({ + "current_stage": "TASK-03", + "size_class": "medium", + "stages": { + "TASK-02": {"status": "completed", "executor": "architect", "retry_count": 0}, + "TASK-03": {"status": "failed", "executor": "developer", "retry_count": 1}, + }, + }), encoding="utf-8") + + second = devflow.resolve_and_diff(state_path, sid, str(project_dir)) + self.assertEqual(len(second["changes"]), 1) + self.assertEqual(second["changes"][0]["stage"], "TASK-03") + self.assertEqual(second["changes"][0]["retry_count"], 1) + self.assertEqual(second["changes"][0]["status"], "failed") + + # 没有变化时，第三次调用应该不再产出 changes。 + third = devflow.resolve_and_diff(state_path, sid, str(project_dir)) + self.assertEqual(third["changes"], []) + finally: + if old_env is None: + os.environ.pop("CODEBUDDY_CONFIG_DIR", None) + else: + os.environ["CODEBUDDY_CONFIG_DIR"] = old_env + + +class PortableSchemaStageSnapshotTests(unittest.TestCase): + """Portable（v2.0）用 `executor_role` 而不是 `executor`，且没有 `review_result`。""" + + def test_reads_executor_role_and_top_level_execution_context(self): + workflow_state = { + "version": "2.0", + "current_stage": "IMPLEMENT", + "size_class": "medium", + "execution_mode": "isolated", + "host_adapter": "codebuddy", + "run_id": "run-abc", + "stages": { + "DESIGN": {"status": "completed", "executor_role": "devflow-architect", "retry_count": 0}, + "IMPLEMENT": {"status": "failed", "executor_role": "devflow-developer", "retry_count": 1}, + }, + } + snap = devflow.stage_snapshot(workflow_state) + self.assertEqual(snap["schema_version"], "2.0") + self.assertEqual(snap["execution_mode"], "isolated") + self.assertEqual(snap["stages"]["DESIGN"]["executor"], "devflow-architect") + self.assertEqual(snap["stages"]["IMPLEMENT"]["retry_count"], 1) + self.assertIsNone(snap["stages"]["IMPLEMENT"]["review_result"]) + + def test_diff_detects_retry_on_portable_shape_same_as_classic(self): + prev = devflow.stage_snapshot({ + "version": "2.0", + "stages": {"IMPLEMENT": {"status": "in_progress", "executor_role": "devflow-developer", "retry_count": 0}}, + }) + curr = devflow.stage_snapshot({ + "version": "2.0", + "stages": {"IMPLEMENT": {"status": "failed", "executor_role": "devflow-developer", "retry_count": 1}}, + }) + changes = devflow.diff_stage_changes(prev, curr) + self.assertEqual(len(changes), 1) + self.assertEqual(changes[0]["stage"], "IMPLEMENT") + self.assertEqual(changes[0]["retry_count"], 1) + self.assertEqual(changes[0]["executor"], "devflow-developer") + + +class ArtifactsScanFallbackTests(unittest.TestCase): + """`topology: spawn` 宿主没有 team 目录，靠扫 artifacts/ 兜底发现 task_slug。""" + + def test_no_artifacts_dir_returns_none(self): + with tempfile.TemporaryDirectory() as td: + self.assertIsNone(devflow._scan_artifacts_for_active_run(td)) + + def test_ignores_non_devflow_json_and_picks_in_progress_run(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + artifacts = root / "artifacts" + + # 不是 devflow 产物的 JSON（没有 stages 字段），不能被误当成一次运行。 + noise_dir = artifacts / "not-a-devflow-run" + noise_dir.mkdir(parents=True) + (noise_dir / "workflow-state.json").write_text(json.dumps({"hello": "world"}), encoding="utf-8") + + done_dir = artifacts / "finished-task_20260910_0900" + done_dir.mkdir(parents=True) + (done_dir / "workflow-state.json").write_text(json.dumps({ + "status": "completed", "stages": {"DESIGN": {"status": "completed"}}, + }), encoding="utf-8") + + active_dir = artifacts / "active-task_20260911_1000" + active_dir.mkdir(parents=True) + (active_dir / "workflow-state.json").write_text(json.dumps({ + "status": "in_progress", "stages": {"IMPLEMENT": {"status": "in_progress"}}, + }), encoding="utf-8") + + result = devflow._scan_artifacts_for_active_run(str(root)) + self.assertIsNotNone(result) + self.assertEqual(result["task_slug"], "active-task_20260911_1000") + self.assertIsNone(result["team_dir"]) + + def test_finished_detection_works_for_classic_schema_without_top_level_status(self): + """Classic（v1.3）没有顶层 status 字段，"跑完没跑完"只能看 last_event。""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) + artifacts = root / "artifacts" + + done_dir = artifacts / "classic-done_20260910_0900" + done_dir.mkdir(parents=True) + (done_dir / "workflow-state.json").write_text(json.dumps({ + "version": "1.3", "last_event": "workflow_completed", + "stages": {"SOLO": {"status": "completed"}}, + }), encoding="utf-8") + + active_dir = artifacts / "classic-active_20260911_1000" + active_dir.mkdir(parents=True) + (active_dir / "workflow-state.json").write_text(json.dumps({ + "version": "1.3", "last_event": "TASK-02_completed", + "stages": {"TASK-02": {"status": "completed"}, "TASK-03": {"status": "in_progress"}}, + }), encoding="utf-8") + + result = devflow._scan_artifacts_for_active_run(str(root)) + self.assertIsNotNone(result) + self.assertEqual(result["task_slug"], "classic-active_20260911_1000") + + def test_resolve_and_diff_switches_task_slug_when_a_newer_run_appears(self): + """真实 bug 回归测试：同一个长生命周期 sid 先后归属两次不同的 devflow 运行 + （没有真正 team_create 时的降级场景——第二次 `/start-devflow` 复用了同一个 + session）。第一次探测缓存下的 task_slug 不能在第二次运行开始后继续沿用； + 必须切换到新的那个，而且两个 task_slug 各自的 diff 历史不能互相污染。""" + with tempfile.TemporaryDirectory() as td: + project_dir = Path(td) / "project" + state_path = project_dir / ".codebuddy" / "skills" / "agent-observability" / "logs" / ".state.json" + sid = "sid-reused-across-two-runs" + + first_dir = project_dir / "artifacts" / "first-task_20260914_0900" + first_dir.mkdir(parents=True) + (first_dir / "workflow-state.json").write_text(json.dumps({ + "version": "1.3", "last_event": "workflow_completed", "current_stage": "SOLO", + "stages": {"SOLO": {"status": "completed", "retry_count": 0}}, + }), encoding="utf-8") + + first = devflow.resolve_and_diff(state_path, sid, str(project_dir)) + self.assertEqual(first["task_slug"], "first-task_20260914_0900") + + # 同一个 sid，第二次运行出现，且比第一次更新（mtime 更新）。 + import time + time.sleep(0.01) + second_dir = project_dir / "artifacts" / "second-task_20260914_1100" + second_dir.mkdir(parents=True) + (second_dir / "workflow-state.json").write_text(json.dumps({ + "version": "1.3", "last_event": "workflow_completed", "current_stage": "SOLO", + "stages": {"SOLO": {"status": "completed", "retry_count": 1}}, + }), encoding="utf-8") + + second = devflow.resolve_and_diff(state_path, sid, str(project_dir)) + self.assertEqual(second["task_slug"], "second-task_20260914_1100") + self.assertEqual(second["changes"], []) # 对 second 是首次观测，不产出变更 + + # 第一个 task_slug 的历史没有被污染：如果它重新变成"最新"（比如被再次修改）， + # 应该正确切回，并且不会把 second 的历史错当成 first 的基线。 + (first_dir / "workflow-state.json").write_text(json.dumps({ + "version": "1.3", "last_event": "workflow_completed", "current_stage": "SOLO", + "stages": {"SOLO": {"status": "completed", "retry_count": 1}}, + }), encoding="utf-8") + time.sleep(0.01) + first_dir.joinpath("workflow-state.json").touch() + + third = devflow.resolve_and_diff(state_path, sid, str(project_dir)) + self.assertEqual(third["task_slug"], "first-task_20260914_0900") + # first 上一次被观测到时 retry_count 还是 0，现在变成 1——应该被识别为变化， + # 而不是被 second 的快照历史污染成"首次观测"或者对不上的 diff。 + self.assertEqual(len(third["changes"]), 1) + self.assertEqual(third["changes"][0]["retry_count"], 1) + + def test_resolve_and_diff_end_to_end_via_artifacts_scan_no_team(self): + with tempfile.TemporaryDirectory() as td: + project_dir = Path(td) / "project" + artifacts_dir = project_dir / "artifacts" / "portable-task_20260911_1100" + artifacts_dir.mkdir(parents=True) + state_file = artifacts_dir / "workflow-state.json" + state_file.write_text(json.dumps({ + "version": "2.0", + "status": "in_progress", + "current_stage": "REVIEW", + "execution_mode": "isolated", + "stages": {"REVIEW": {"status": "in_progress", "executor_role": "devflow-code-reviewer", "retry_count": 0}}, + }), encoding="utf-8") + + import os + old_env = os.environ.get("CODEBUDDY_CONFIG_DIR") + os.environ["CODEBUDDY_CONFIG_DIR"] = str(Path(td) / "empty-codebuddy-home") + try: + state_path = project_dir / ".codebuddy" / "skills" / "agent-observability" / "logs" / ".state.json" + sid = "sid-spawn-1" + + first = devflow.resolve_and_diff(state_path, sid, str(project_dir)) + self.assertIsNotNone(first) + self.assertEqual(first["task_slug"], "portable-task_20260911_1100") + self.assertEqual(first["schema_version"], "2.0") + self.assertEqual(first["execution_mode"], "isolated") + self.assertEqual(first["changes"], []) + + state_file.write_text(json.dumps({ + "version": "2.0", + "status": "in_progress", + "current_stage": "REVIEW", + "execution_mode": "isolated", + "stages": {"REVIEW": {"status": "failed", "executor_role": "devflow-code-reviewer", "retry_count": 1}}, + }), encoding="utf-8") + + second = devflow.resolve_and_diff(state_path, sid, str(project_dir)) + self.assertEqual(len(second["changes"]), 1) + self.assertEqual(second["changes"][0]["retry_count"], 1) + self.assertEqual(second["changes"][0]["status"], "failed") + finally: + if old_env is None: + os.environ.pop("CODEBUDDY_CONFIG_DIR", None) + else: + os.environ["CODEBUDDY_CONFIG_DIR"] = old_env + + +if __name__ == "__main__": + unittest.main() diff --git a/.claude/skills/agent-observability/tests/test_pricing_overrides.py b/.claude/skills/agent-observability/tests/test_pricing_overrides.py new file mode 100644 index 0000000..75615a3 --- /dev/null +++ b/.claude/skills/agent-observability/tests/test_pricing_overrides.py @@ -0,0 +1,394 @@ +"""自定义模型定价覆盖 + 看板未定价模型提示。 + +覆盖文件是**人手写的常驻配置**，而且加载失败要静默降级——这类"错了也不许 +报错"的代码最容易在半年后被悄悄改坏（比如有人顺手把 try/except 去掉、把合并 +改成整 key 覆盖、或让降级路径开始 print）。这里把字段级合并、四种/五种降级 +路径、缓存语义、未定价口径、看板聚合与"不重算历史成本"逐条钉住。 + +两个容易踩的隔离坑，本文件统一在 setUp/tearDown 处理： +1. `load_prices()` 带 `lru_cache`，不 `clear_price_cache()` 用例之间会串味； +2. `AOBS_PRICING_OVERRIDES_PATH` 是进程级环境变量，不还原会污染同进程里 + 其它测试模块（尤其是会真的去读默认覆盖文件的 hook 链路）。 + +真实日志对账（hy4-preview-ioa / hy3-ioa 的计数）不写进单测：日志会持续增长， +写死数字会变成 flaky 用例。那部分以人工冒烟的方式在 TASK-04 报告里记录。 +""" +from __future__ import annotations + +import contextlib +import io +import json +import os +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "scripts" +if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) + +import build_dashboard_data as bdd # type: ignore +from core import emitter # type: ignore + +OVERRIDES_ENV = "AOBS_PRICING_OVERRIDES_PATH" + + +class _OverridesTestCase(unittest.TestCase): + """统一处理环境变量与 `load_prices()` 缓存的隔离。""" + + def setUp(self): + self._env_backup = os.environ.get(OVERRIDES_ENV) + os.environ.pop(OVERRIDES_ENV, None) + emitter.clear_price_cache() + self._tmp = tempfile.TemporaryDirectory() + self.tmp = Path(self._tmp.name) + + def tearDown(self): + # 顺序要紧：先还原 env，再清缓存，最后清目录，避免留下一个指向已删除 + # 目录的环境变量给下一个用例。 + if self._env_backup is None: + os.environ.pop(OVERRIDES_ENV, None) + else: + os.environ[OVERRIDES_ENV] = self._env_backup + emitter.clear_price_cache() + self._tmp.cleanup() + + def use_overrides(self, payload, name: str = "overrides.json") -> Path: + """把 payload 写成覆盖文件并指向它（JSON 文本原样写入，便于构造非法输入）。""" + path = self.tmp / name + text = payload if isinstance(payload, str) else json.dumps(payload) + path.write_text(text, encoding="utf-8") + os.environ[OVERRIDES_ENV] = str(path) + emitter.clear_price_cache() + return path + + def pure_builtin(self) -> dict: + """"纯内置表"的唯一权威来源——不把 pricing.json 的内容抄进断言里， + 否则内置表一调整这里就假红；但内置表本身必须是非空的，否则下面的 + "降级后 == 纯内置表" 断言会退化成空表比空表。""" + builtin = emitter._load_builtin_prices() + self.assertTrue(builtin, "内置价格表不应为空，否则降级断言失去意义") + self.assertIn("gpt-4o", builtin) + return builtin + + +class OverridesPathResolutionTests(_OverridesTestCase): + """AC4：路径可覆盖。默认位置刻意放在 skills 树**外**—— + `scripts/build-classic-hosts.py` 会整棵同步 `.codebuddy/skills`，放树内会让 + 用户每次改价都产生一次 `--check` drift，并被复制进 .claude/.cursor 宿主包。""" + + def test_env_var_wins_over_default(self): + path = self.use_overrides({"zz-model": {"output": 1.0}}) + self.assertEqual(emitter.resolve_overrides_path(), path) + + def test_tilde_is_expanded(self): + os.environ[OVERRIDES_ENV] = "~/aobs-overrides-test.json" + emitter.clear_price_cache() + self.assertEqual(emitter.resolve_overrides_path(), Path.home() / "aobs-overrides-test.json") + + def test_unset_falls_back_to_default_path(self): + self.assertEqual(emitter.resolve_overrides_path(), emitter.DEFAULT_OVERRIDES_PATH) + + def test_blank_env_falls_back_to_default(self): + # 空字符串/纯空白都等同于"没设置"，不能变成指向 CWD 的相对路径。 + os.environ[OVERRIDES_ENV] = " " + emitter.clear_price_cache() + self.assertEqual(emitter.resolve_overrides_path(), emitter.DEFAULT_OVERRIDES_PATH) + + def test_default_path_is_outside_the_skills_tree(self): + default = emitter.DEFAULT_OVERRIDES_PATH + self.assertIsNotNone(default) + self.assertEqual( + default, + ROOT.parents[2] / ".codebuddy" / "agent-observability" / "pricing.overrides.json", + ) + self.assertNotIn("skills", default.parts[-3:]) + self.assertFalse(str(default).startswith(str(ROOT))) + + +class MergePricesTests(unittest.TestCase): + """AC2：字段级合并。整 key 覆盖被刻意否决过——漏写字段会把该字段按 0 计， + 静默把成本算没，风险高于收益。""" + + def test_explicit_field_overrides_builtin_while_others_are_kept(self): + merged = emitter.merge_prices( + {"gpt-4o": {"input": 2.5, "output": 10.0, "cache_read": 1.25}}, + {"gpt-4o": {"output": 99.0}}, + ) + self.assertEqual(merged["gpt-4o"]["output"], 99.0) + self.assertEqual(merged["gpt-4o"]["input"], 2.5) + self.assertEqual(merged["gpt-4o"]["cache_read"], 1.25) + + def test_new_model_can_be_added(self): + merged = emitter.merge_prices({"gpt-4o": {"output": 10.0}}, {"zz-new": {"output": 9.0}}) + self.assertEqual(merged["zz-new"], {"output": 9.0}) + + def test_merge_does_not_mutate_its_arguments(self): + base = {"gpt-4o": {"input": 2.5, "output": 10.0}} + overrides = {"gpt-4o": {"output": 99.0}} + merged = emitter.merge_prices(base, overrides) + merged["gpt-4o"]["input"] = 0.0 + # 内置表是模块级共享数据（lru_cache 之外还有调用方持有引用）， + # 被就地改过的话一次合并会污染整条 hook 链路。 + self.assertEqual(base["gpt-4o"], {"input": 2.5, "output": 10.0}) + self.assertEqual(overrides["gpt-4o"], {"output": 99.0}) + + def test_empty_or_none_overrides_return_base_copy(self): + base = {"gpt-4o": {"output": 10.0}} + self.assertEqual(emitter.merge_prices(base, None), base) + self.assertEqual(emitter.merge_prices(base, {}), base) + + def test_none_base_returns_overrides_only(self): + self.assertEqual(emitter.merge_prices(None, {"zz-new": {"output": 1.0}}), {"zz-new": {"output": 1.0}}) + + def test_non_dict_rows_are_ignored(self): + merged = emitter.merge_prices({"gpt-4o": {"output": 10.0}}, {"zz-bad": 9.0, "zz-ok": {"output": 2.0}}) + self.assertNotIn("zz-bad", merged) + self.assertEqual(merged["zz-ok"], {"output": 2.0}) + + +class LoadPricesOverrideTests(_OverridesTestCase): + """AC2 + AC3：加载与降级。""" + + def test_override_applies_field_level_and_adds_model(self): + self.use_overrides({"zz-custom": {"input": 1.0, "output": 9.0}, "gpt-4o": {"output": 99.0}}) + prices = emitter.load_prices() + self.assertEqual(prices["zz-custom"], {"input": 1.0, "output": 9.0}) + self.assertEqual(prices["gpt-4o"]["output"], 99.0) + self.assertEqual(prices["gpt-4o"]["input"], self.pure_builtin()["gpt-4o"]["input"]) + + def test_missing_file_degrades_to_builtin(self): + os.environ[OVERRIDES_ENV] = str(self.tmp / "does-not-exist.json") + emitter.clear_price_cache() + self.assertEqual(emitter.load_prices(), self.pure_builtin()) + + def test_invalid_json_degrades_to_builtin(self): + self.use_overrides('{"gpt-4o": {"output": 99.0}') + self.assertEqual(emitter.load_prices(), self.pure_builtin()) + + def test_top_level_non_dict_degrades_to_builtin(self): + self.use_overrides("[1, 2, 3]") + self.assertEqual(emitter.load_prices(), self.pure_builtin()) + + def test_non_numeric_field_is_skipped_but_siblings_apply(self): + self.use_overrides({"gpt-4o": {"output": "abc", "input": 3.0}}) + prices = emitter.load_prices() + # 写错一个字段不该把同模型其它字段、其它模型的正确覆盖一起丢掉。 + self.assertEqual(prices["gpt-4o"]["input"], 3.0) + self.assertEqual(prices["gpt-4o"]["output"], self.pure_builtin()["gpt-4o"]["output"]) + + def test_model_with_only_bad_fields_is_dropped_and_stays_unpriced(self): + """计划外决策（已在 code review 接受）：全坏字段的模型不落表。 + 若落表成 `{}`，它既算不出成本又不算"未定价"，看板上会变成一个查不到 + 原因的空洞；丢弃后至少能在"模型定价"提醒里暴露出来。""" + self.use_overrides({"zz-all-bad": {"output": "abc"}}) + prices = emitter.load_prices() + self.assertNotIn("zz-all-bad", prices) + self.assertTrue(emitter.is_unpriced("zz-all-bad")) + + def test_bool_is_not_treated_as_a_price(self): + # bool 是 int 的子类，float(True) == 1.0 会静默变成"单价 1 美元"。 + self.use_overrides({"gpt-4o": {"output": True}}) + prices = emitter.load_prices() + self.assertEqual(prices["gpt-4o"]["output"], self.pure_builtin()["gpt-4o"]["output"]) + + def test_every_degradation_path_is_silent(self): + """静默是刻意的：hook 每次都是新进程，一旦因格式问题 print/抛异常， + 整条 hook 链路都会变得不可用。""" + for payload in ('{"broken": ', "[1,2,3]", '{"gpt-4o": {"output": "abc"}}', '{"gpt-4o": 1}'): + with self.subTest(payload=payload): + self.use_overrides(payload) + out, err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + prices = emitter.load_prices() + self.assertEqual(prices, self.pure_builtin()) + self.assertEqual(out.getvalue(), "") + self.assertEqual(err.getvalue(), "") + + def test_load_prices_is_cached_until_cleared(self): + path = self.use_overrides({"zz-cache-model": {"output": 1.0}}) + self.assertEqual(emitter.load_prices()["zz-cache-model"]["output"], 1.0) + path.write_text(json.dumps({"zz-cache-model": {"output": 2.0}}), encoding="utf-8") + # 未清缓存：同进程内仍是旧值（hook 是短命进程，这个行为是可接受的）。 + self.assertEqual(emitter.load_prices()["zz-cache-model"]["output"], 1.0) + emitter.clear_price_cache() + self.assertEqual(emitter.load_prices()["zz-cache-model"]["output"], 2.0) + + def test_clear_price_cache_picks_up_env_change(self): + self.assertTrue(hasattr(emitter.load_prices, "cache_clear")) + self.use_overrides({"zz-a": {"output": 1.0}}) + self.assertIn("zz-a", emitter.load_prices()) + self.use_overrides({"zz-b": {"output": 2.0}}, name="second.json") + emitter.clear_price_cache() + prices = emitter.load_prices() + self.assertIn("zz-b", prices) + self.assertNotIn("zz-a", prices) + + +class IsUnpricedTests(_OverridesTestCase): + """D4：未定价的唯一口径是 `lookup_price(...) is None`。 + `lookup_price` 会做"最长子串"模糊兜底，所以 `gpt-4o-2024-11-20` 能算出成本， + 不该出现在"建议补价"里——口径一旦漂移，用户会去给明明已经命中的模型补价。""" + + def test_unknown_model_is_unpriced(self): + self.assertTrue(emitter.is_unpriced("zz-totally-unknown")) + + def test_fuzzy_substring_match_is_not_unpriced(self): + self.assertFalse(emitter.is_unpriced("gpt-4o-2024-11-20")) + + def test_missing_or_blank_model_is_unpriced(self): + for model in (None, "", " "): + with self.subTest(model=model): + self.assertTrue(emitter.is_unpriced(model)) + + def test_override_makes_a_model_priced(self): + self.assertTrue(emitter.is_unpriced("zz-then-priced")) + self.use_overrides({"zz-then-priced": {"output": 9.0}}) + self.assertFalse(emitter.is_unpriced("zz-then-priced")) + + +class BuildUnpricedModelsTests(_OverridesTestCase): + """AC5：只看板**统计**，绝不回头重算 cost_usd（D3：覆盖只对新事件生效）。""" + + def _usage(self, model, ts=1.0): + return {"event": "usage", "sid": "s1", "ts": ts, "model": model, "tokens": {"input": 1, "output": 1}} + + def test_counts_only_unpriced_usage_events(self): + events = [ + self._usage("zz-unpriced-a", 1.0), + self._usage("zz-unpriced-a", 2.0), + self._usage("zz-unpriced-b", 3.0), + self._usage("gpt-4o-2024-11-20", 4.0), # 模糊命中，不算未定价 + {"event": "usage", "sid": "s1", "ts": 5.0, "tokens": {"input": 1}}, # model 缺失 + self._usage(" ", 6.0), # model 空白 + {"event": "tool", "sid": "s1", "ts": 7.0, "tool": "Bash", "ms": 1}, # 非 usage + ] + rows = bdd.build_unpriced_models(events) + self.assertEqual(rows, [ + {"name": "zz-unpriced-a", "usageEvents": 2}, + {"name": "zz-unpriced-b", "usageEvents": 1}, + ]) + + def test_empty_input_returns_empty_list(self): + self.assertEqual(bdd.build_unpriced_models([]), []) + + def test_sorted_by_event_count_desc_then_name(self): + events = ( + [self._usage("zz-b", float(i)) for i in range(2)] + + [self._usage("zz-a", 9.0)] + + [self._usage("zz-c", float(i)) for i in range(2)] + ) + rows = bdd.build_unpriced_models(events) + # 计数相同的 zz-b / zz-c 必须按名字定序，否则两次生成的快照无法逐字节比对。 + self.assertEqual([(r["name"], r["usageEvents"]) for r in rows], + [("zz-b", 2), ("zz-c", 2), ("zz-a", 1)]) + + def test_pricing_a_model_removes_it_from_the_list(self): + events = [self._usage("zz-unpriced-a", 1.0), self._usage("zz-unpriced-b", 2.0)] + self.assertEqual(len(bdd.build_unpriced_models(events)), 2) + self.use_overrides({"zz-unpriced-a": {"output": 9.0}}) + rows = bdd.build_unpriced_models(events) + self.assertEqual(rows, [{"name": "zz-unpriced-b", "usageEvents": 1}]) + + def test_emitter_import_failure_degrades_to_empty_list(self): + """目标机可能没装 emitter 的间接依赖；此时只该退化"未定价"这一项， + 看板仍要能出片。""" + with mock.patch.object(bdd, "em", None): + self.assertEqual(bdd.build_unpriced_models([self._usage("zz-unpriced-a")]), []) + + +class MainSnapshotTests(_OverridesTestCase): + """main() 级别：键位、空列表、不重算成本、无覆盖时与改动前一致（AC1）。""" + + def _events(self): + return [ + {"event": "user_prompt_submit", "sid": "s1", "ts": 1.0, "turn_id": "t1", "agent": "main"}, + {"event": "usage", "sid": "s1", "ts": 2.0, "model": "gpt-4o-2024-11-20", + "tokens": {"input": 100, "output": 10}, "cost_usd": 0.0035, "agent": "main"}, + {"event": "usage", "sid": "s1", "ts": 3.0, "model": "zz-unpriced-a", + "tokens": {"input": 100, "output": 10}, "agent": "main"}, + {"event": "usage", "sid": "s1", "ts": 4.0, "model": "zz-unpriced-a", + "tokens": {"input": 100, "output": 10}, "agent": "main"}, + ] + + def _run_main(self, root: Path, out_name: str) -> tuple[dict, str]: + metrics = root / "metrics.ndjson" + metrics.write_text("\n".join(json.dumps(e) for e in self._events()) + "\n", encoding="utf-8") + state = root / ".state.json" + state.write_text("{}", encoding="utf-8") + out = root / out_name + old_argv = sys.argv + sys.argv = ["build_dashboard_data.py", "--project-root", str(root), + "--out", str(out), "--metrics-path", str(metrics), "--state-path", str(state)] + buf = io.StringIO() + try: + with contextlib.redirect_stdout(buf): + rc = bdd.main() + finally: + sys.argv = old_argv + self.assertEqual(rc, 0) + self.assertIn("wrote ", buf.getvalue()) + return json.loads(out.read_text("utf-8")), buf.getvalue() + + def test_snapshot_exposes_unpriced_models_right_after_model_costs(self): + with tempfile.TemporaryDirectory() as td: + data, _ = self._run_main(Path(td), "out.json") + keys = list(data.keys()) + self.assertEqual(keys[keys.index("modelCosts") + 1], "unpricedModels") + self.assertEqual(data["unpricedModels"], [{"name": "zz-unpriced-a", "usageEvents": 2}]) + self.assertEqual([m["name"] for m in data["modelCosts"]], ["gpt-4o-2024-11-20"]) + + def test_no_unpriced_models_yields_empty_list_not_null(self): + with tempfile.TemporaryDirectory() as td: + self.use_overrides({"zz-unpriced-a": {"output": 9.0}}) + data, _ = self._run_main(Path(td), "out.json") + self.assertEqual(data["unpricedModels"], []) + + def test_override_never_recomputes_already_written_costs(self): + """D3 的回归护栏：给未定价模型补价后，只有 unpricedModels 会变， + modelCosts / daily / sessions 必须逐字节不变（历史成本不回溯）。""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) + before, _ = self._run_main(root, "before.json") + self.use_overrides({"zz-unpriced-a": {"output": 9.0}}) + after, _ = self._run_main(root, "after.json") + before.pop("generated_at") + after.pop("generated_at") + self.assertEqual(before["unpricedModels"], [{"name": "zz-unpriced-a", "usageEvents": 2}]) + self.assertEqual(after["unpricedModels"], []) + before.pop("unpricedModels") + after.pop("unpricedModels") + self.assertEqual(before, after) + + def test_broken_override_file_keeps_cli_at_exit_zero_and_silent(self): + """AC3 的 CLI 层护栏：覆盖文件坏掉时看板仍然出片（exit 0、stderr 为空）， + 而不是把整条构建链路搞挂。""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) + self.use_overrides('{"gpt-4o": {"output": 99.0}') # 故意截断的非法 JSON + err = io.StringIO() + with contextlib.redirect_stderr(err): + data, _ = self._run_main(root, "out.json") + self.assertEqual(err.getvalue(), "") + self.assertEqual(data["unpricedModels"], [{"name": "zz-unpriced-a", "usageEvents": 2}]) + + def test_no_override_file_matches_nonexistent_override_path(self): + """AC1 的核心回归：不传覆盖文件时价格表就是纯内置表， + 与"指向一个不存在的路径"的输出完全一致（忽略 generated_at）。""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) + unset, _ = self._run_main(root, "unset.json") + os.environ[OVERRIDES_ENV] = str(root / "nope.json") + emitter.clear_price_cache() + missing, _ = self._run_main(root, "missing.json") + unset.pop("generated_at") + missing.pop("generated_at") + self.assertEqual(unset, missing) + self.assertEqual(unset["unpricedModels"], [{"name": "zz-unpriced-a", "usageEvents": 2}]) + + +if __name__ == "__main__": + unittest.main() diff --git a/.claude/skills/agent-observability/tests/test_runtime_flow.py b/.claude/skills/agent-observability/tests/test_runtime_flow.py new file mode 100644 index 0000000..7bd143b --- /dev/null +++ b/.claude/skills/agent-observability/tests/test_runtime_flow.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path +from unittest import mock +import sys + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "scripts" +if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) + +from core import runtime # type: ignore + + +class RuntimeFlowTests(unittest.TestCase): + def test_resolve_active_agent_prefers_inbox_inferred_agent_when_available(self): + data = { + "tool_name": "Task", + "tool_input": {"subagent_name": "developer"}, + } + with tempfile.TemporaryDirectory() as td: + state_path = Path(td) / ".state.json" + with mock.patch.object(runtime.agent_identity, "merge_agent_identity_from_inbox", return_value=("qa", "developer")) as merge_identity: + active, dispatched = runtime.agent_identity.resolve_active_agent_for_event( + state_path=state_path, + sid="s-inbox", + cwd=td, + data=data, + ) + + self.assertEqual(active, "qa") + self.assertEqual(dispatched, "developer") + merge_identity.assert_called_once() + + def test_handle_post_routes_through_expected_collaborators(self): + data = { + "session_id": "s-runtime", + "cwd": "/tmp/demo", + "tool_name": "Read", + "transcript_path": "/tmp/demo.jsonl", + } + with tempfile.TemporaryDirectory() as td: + with mock.patch.object(runtime, "build_runtime_paths") as mock_paths, \ + mock.patch.object(runtime.collector, "record_post", return_value=12), \ + mock.patch.object(runtime.collector, "load_cached_inventory", return_value=({}, {})), \ + mock.patch.object(runtime.scanner, "scan_skills_and_rules", return_value=({}, {})), \ + mock.patch.object(runtime.collector, "cache_inventory"), \ + mock.patch.object(runtime.collector, "record_tool_usage", return_value=([], [])), \ + mock.patch.object(runtime.agent_identity, "resolve_active_agent_for_event", return_value=("main", None)), \ + mock.patch.object(runtime, "current_turn_id", return_value="turn-1"), \ + mock.patch.object(runtime.collector, "extract_tool_call_id", return_value=None), \ + mock.patch.object(runtime, "flush_pending_tool_events"), \ + mock.patch.object(runtime.collector, "find_current_tool_context", return_value=None), \ + mock.patch.object(runtime.collector, "related_transcript_paths", return_value=["/tmp/demo.jsonl"]), \ + mock.patch.object(runtime, "emit_transcript_events", return_value=[]), \ + mock.patch.object(runtime.collector, "find_fallback_usage_event", return_value=None), \ + mock.patch.object(runtime.st, "append_pending_tool_emit"), \ + mock.patch.object(runtime.agentlens, "emit_post_step"): + mock_paths.return_value = runtime.RuntimePaths( + base_dir=Path(td), + state_path=Path(td) / ".state.json", + pending_path=Path(td) / ".pending.json", + log_path=Path(td) / "metrics.ndjson", + ) + runtime.handle_post(data) + + def test_main_dispatches_supported_phase_to_handler(self): + with mock.patch.object(runtime, "handle_post") as handle_post, \ + mock.patch.object(runtime.collector, "read_stdin_json", return_value={"session_id": "s1"}): + rc = runtime.main(["post"]) + self.assertEqual(rc, 0) + handle_post.assert_called_once() diff --git a/.codebuddy/runtime/workflow-state-spec.md b/.codebuddy/runtime/workflow-state-spec.md index a826c2e..264e306 100644 --- a/.codebuddy/runtime/workflow-state-spec.md +++ b/.codebuddy/runtime/workflow-state-spec.md @@ -118,6 +118,8 @@ failed → in_progress（重试） **Leader 汇总时**：`summary.status="completed"` + `summary.report_path` + `summary.completed_at=now` +**任一角色判定自身失败时** ⭐：`status="failed"` + `stages[当前阶段].retry_count = (原值 or 0) + 1` + `last_error=<失败原因>` + `completed_at=now` + `last_event="TASK-XX_failed"`。`retry_count` 只能由失败角色自己在这一步递增，不得由 Main Agent 代写；Main Agent 在调度自检时只读取该值判断重试或暂停（见"调度自检"章节），不修改它。 + ## 初始化协议 Main Agent 从模板创建 → 填充 `task_id`/`task_slug`/`run_mode`/`runtime_mode`/`project_config` → Phase 0 3 维度打分写入 `size_class` → `current_stage="PHASE-0"` → `last_event="workflow_initialized"` diff --git a/.codebuddy/settings.json b/.codebuddy/settings.json index 84f2a6a..a1238c1 100644 --- a/.codebuddy/settings.json +++ b/.codebuddy/settings.json @@ -1,5 +1,27 @@ { "hooks": { + "SessionStart": [ + { + "hooks": [ + { + "type": "command", + "command": "\"$CODEBUDDY_PROJECT_DIR\"/.codebuddy/skills/agent-observability/scripts/run_hook.sh session-start", + "timeout": 5 + } + ] + } + ], + "UserPromptSubmit": [ + { + "hooks": [ + { + "type": "command", + "command": "\"$CODEBUDDY_PROJECT_DIR\"/.codebuddy/skills/agent-observability/scripts/run_hook.sh user-prompt-submit", + "timeout": 5 + } + ] + } + ], "PreToolUse": [ { "matcher": "Bash", @@ -12,7 +34,7 @@ ] }, { - "matcher": "send_message", + "matcher": "SendMessage", "hooks": [ { "type": "command", @@ -20,6 +42,39 @@ "timeout": 10 } ] + }, + { + "matcher": "", + "hooks": [ + { + "type": "command", + "command": "\"$CODEBUDDY_PROJECT_DIR\"/.codebuddy/skills/agent-observability/scripts/run_hook.sh pre", + "timeout": 5 + } + ] + } + ], + "PostToolUse": [ + { + "matcher": "", + "hooks": [ + { + "type": "command", + "command": "\"$CODEBUDDY_PROJECT_DIR\"/.codebuddy/skills/agent-observability/scripts/run_hook.sh post", + "timeout": 30 + } + ] + } + ], + "Stop": [ + { + "hooks": [ + { + "type": "command", + "command": "\"$CODEBUDDY_PROJECT_DIR\"/.codebuddy/skills/agent-observability/scripts/run_hook.sh stop", + "timeout": 30 + } + ] } ] } diff --git a/.codebuddy/skills/agent-observability/SKILL.md b/.codebuddy/skills/agent-observability/SKILL.md new file mode 100644 index 0000000..8bff9f0 --- /dev/null +++ b/.codebuddy/skills/agent-observability/SKILL.md @@ -0,0 +1,33 @@ +--- +name: agent-observability +description: Add or maintain CodeBuddy hook observability for projects that need local metrics for tool latency, transcript token usage, session cost, and multi-agent trace attribution. Use when wiring hook-based telemetry, debugging missing usage events, and validating AgentLens state. +--- + +## Workflow + +1. Merge `templates/settings-hook.json` into the user or project `settings.json`. +2. Keep hook entries command-only and keep timeouts around 5 to 10 seconds. +3. Run a normal CodeBuddy session and inspect `logs/metrics.ndjson`. +4. Inspect `logs/.state.json` when usage replay, offsets, or AgentLens turn state look wrong. +5. Keep the runtime generic and verify output from `metrics.ndjson` and `.state.json`. + +## Read Next + +- Read `references/quickstart.md` for setup and smoke-test steps. +- Read `references/schema-v2.md` when you need field definitions or sidecar state shape. +## Key Files + +- `scripts/main.py`: hook entrypoint for `session-start`, `user-prompt-submit`, `pre`, `post`, and `stop` +- `scripts/run_hook.sh`: stable shell wrapper for CodeBuddy hook commands +- `scripts/core/collector.py`: transcript parsing, pre/post pairing, and session usage recording +- `scripts/core/agentlens.py`: AgentLens trace emission and agent or step grouping +- `scripts/core/state.py`: persisted hook state, offsets, and AgentLens sidecar state +- `scripts/core/devflow.py`: optional devflow-awareness — detects a `multi-agents-devflow-*` team, reads `workflow-state.json`, and emits `stage_transition` events. No-ops entirely on non-devflow projects. +- `templates/settings-hook.json`: hook wiring template + +## Output Contract + +- Always emit `event`, `sid`, and `ts`. +- Emit `tool` for per-call latency and `usage` for transcript-derived token and cost events. +- Emit `stop` for end-of-session flush. +- Never block the main hook flow; hook exits must stay `0`. diff --git a/.codebuddy/skills/agent-observability/config/pricing.json b/.codebuddy/skills/agent-observability/config/pricing.json new file mode 100644 index 0000000..59d8e58 --- /dev/null +++ b/.codebuddy/skills/agent-observability/config/pricing.json @@ -0,0 +1,39 @@ +{ + "claude-opus-4.8": {"input": 5.0, "output": 25.0, "cache_read": 0.5, "cache_write": 6.25}, + "claude-opus-4.7": {"input": 5.0, "output": 25.0, "cache_read": 0.5, "cache_write": 6.25}, + "claude-opus-4.6": {"input": 5.0, "output": 25.0, "cache_read": 0.5, "cache_write": 6.25}, + "claude-opus-4": {"input": 5.0, "output": 25.0, "cache_read": 0.5, "cache_write": 6.25}, + "claude-sonnet-4.6": {"input": 3.0, "output": 15.0, "cache_read": 0.3, "cache_write": 3.75}, + "claude-sonnet-4.5": {"input": 3.0, "output": 15.0, "cache_read": 0.3, "cache_write": 3.75}, + "claude-sonnet-4": {"input": 3.0, "output": 15.0, "cache_read": 0.3, "cache_write": 3.75}, + "claude-3-7-sonnet": {"input": 3.0, "output": 15.0, "cache_read": 0.3, "cache_write": 3.75}, + "claude-3-5-sonnet": {"input": 3.0, "output": 15.0, "cache_read": 0.3, "cache_write": 3.75}, + "claude-haiku-4.5": {"input": 1.0, "output": 5.0, "cache_read": 0.1, "cache_write": 1.25}, + "claude-3-5-haiku": {"input": 1.0, "output": 5.0, "cache_read": 0.1, "cache_write": 1.25}, + "claude-haiku-4": {"input": 1.0, "output": 5.0, "cache_read": 0.1, "cache_write": 1.25}, + "gpt-5.5-fast": {"input": 12.5, "output": 75.0, "cache_read": 1.25, "cache_write": 12.5}, + "gpt-5.5": {"input": 5.0, "output": 30.0, "cache_read": 0.5, "cache_write": 5.0}, + "gpt-5.4-fast": {"input": 5.0, "output": 30.0, "cache_read": 0.5, "cache_write": 5.0}, + "gpt-5.4": {"input": 2.5, "output": 15.0, "cache_read": 0.25, "cache_write": 2.5}, + "gpt-5.3-codex": {"input": 1.75, "output": 14.0, "cache_read": 0.175, "cache_write": 1.75}, + "gpt-4o": {"input": 2.5, "output": 10.0, "cache_read": 1.25, "cache_write": 2.5}, + "gpt-4o-mini": {"input": 0.15, "output": 0.6, "cache_read": 0.075, "cache_write": 0.15}, + "gemini-3.5-flash": {"input": 1.5, "output": 9.0, "cache_read": 0.15, "cache_write": 1.5}, + "gemini-3.1-pro": {"input": 2.0, "output": 12.0, "cache_read": 0.2, "cache_write": 2.0}, + "gemini-3.1-flash-lite": {"input": 0.25, "output": 1.5, "cache_read": 0.025, "cache_write": 0.25}, + "gemini-3-flash": {"input": 0.5, "output": 3.0, "cache_read": 0.05, "cache_write": 0.5}, + "gemini-2": {"input": 0.5, "output": 3.0, "cache_read": 0.05, "cache_write": 0.5}, + "glm-5.1": {"input": 0.857, "output": 3.429, "cache_read": 0.186, "cache_write": 0.857}, + "glm-5v-turbo": {"input": 0.714, "output": 3.143, "cache_read": 0.171, "cache_write": 0.714}, + "glm-5-turbo": {"input": 0.714, "output": 3.143, "cache_read": 0.171, "cache_write": 0.714}, + "glm-5": {"input": 0.857, "output": 3.429, "cache_read": 0.186, "cache_write": 0.857}, + "kimi-k2.6": {"input": 0.929, "output": 3.857, "cache_read": 0.157, "cache_write": 0.929}, + "kimi-k2.5": {"input": 0.571, "output": 3.0, "cache_read": 0.1, "cache_write": 0.571}, + "kimi": {"input": 0.571, "output": 3.0, "cache_read": 0.1, "cache_write": 0.571}, + "deepseek-v4-pro": {"input": 0.429, "output": 0.857, "cache_read": 0.004, "cache_write": 0.429}, + "deepseek-v4-flash": {"input": 0.143, "output": 0.286, "cache_read": 0.003, "cache_write": 0.143}, + "deepseek-v4": {"input": 0.429, "output": 0.857, "cache_read": 0.004, "cache_write": 0.429}, + "deepseek": {"input": 0.143, "output": 0.286, "cache_read": 0.003, "cache_write": 0.143}, + "minimax-m2.7": {"input": 0.3, "output": 1.2, "cache_read": 0.06, "cache_write": 0.3}, + "minimax": {"input": 0.3, "output": 1.2, "cache_read": 0.06, "cache_write": 0.3} +} diff --git a/.codebuddy/skills/agent-observability/logs/.gitkeep b/.codebuddy/skills/agent-observability/logs/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/.codebuddy/skills/agent-observability/references/quickstart.md b/.codebuddy/skills/agent-observability/references/quickstart.md new file mode 100644 index 0000000..72ea7bf --- /dev/null +++ b/.codebuddy/skills/agent-observability/references/quickstart.md @@ -0,0 +1,195 @@ +## Quickstart + +### 安装 + +将 `templates/settings-hook.json` 合并到 `~/.codebuddy/settings.json` 或 `/.codebuddy/settings.json`。 + +必须启用 hooks：`SessionStart`、`UserPromptSubmit`、`PreToolUse`、`PostToolUse`、`Stop`。 + +### 查看输出 + +完成一次正常会话后检查： +- `.codebuddy/skills/agent-observability/logs/metrics.ndjson` +- `.codebuddy/skills/agent-observability/logs/.state.json` + +### 指定数据源路径（可选） + +`build_dashboard_data.py` 默认从 `/logs/metrics.ndjson` 与 `/logs/.state.json` 读取，**不传参数时行为完全不变**。 + +如需从其它位置读取，可用 `--metrics-path` / `--state-path` 覆盖： + +```bash +python3 scripts/build_dashboard_data.py \ + --metrics-path ~/Downloads/metrics.ndjson \ + --state-path ~/Downloads/.state.json \ + --out dashboard/dashboard-data.json +``` + +注意： + +- 两个参数都留空（或省略）时回退到 `/logs/` 下的默认文件。 +- 路径支持 `~` 展开（如上例的 `~/Downloads/...`）。 +- 输出 `dashboard-data.json` 的 `source.metrics_ndjson` / `source.state_json` 会**如实反映实际读取到的路径**，覆盖后自然指向你给定的文件，便于核对数据来源。 + +重点字段： +- `event=tool`：工具耗时、`skill`、`rule` +- `event=usage`：`tokens`、`model`、`cost_usd` +- `event=stop`：会话尾部 flush 与总成本 + +### 导出最慢的工具调用（可选） + +`--top-slow N` 会在生成 `dashboard-data.json` 的同时，额外在**终端**打印耗时最长的 N 次 `event=tool` 调用（工具名 + 耗时 ms，按耗时降序）。排查"哪次工具调用拖慢了会话"时不用再去翻原始 ndjson。 + +```bash +python3 scripts/build_dashboard_data.py --top-slow 5 +``` + +输出示例（排在 `wrote ...` 之后）： + +``` +top-slow 5 tool calls (by ms): + 1. Bash 4820ms + 2. Grep 1230ms + 3. Read 310ms +``` + +注意： + +- **默认不开启**：不传该参数时不会打印任何额外内容，原有输出一字不变。 +- **只读**：结果只打印到终端，不会写进 `dashboard-data.json`——输出结构与不开时完全一致，看板不受影响。 +- 只统计 `event=tool` 事件；`ms` 缺失或非数字时按 `0` 兜底，与看板其它统计口径一致。 +- `N` 大于实际 tool 事件数时取全部，不报错；没有任何 tool 事件时不打印该段。 + +### 自定义模型定价覆盖（可选） + +内置价格表 `config/pricing.json` 覆盖不到的模型（自部署模型、内部代号、刚发布还没来得及收录的模型）， +`cost_usd` 会算不出来。给这些模型补单价不用改内置文件——写一份**只含差异**的覆盖文件即可。 + +**放哪**：默认 `/.codebuddy/agent-observability/pricing.overrides.json` +（`` 是 `.codebuddy/` 所在的那一层，例如 `/Users/me/my-project`）。 + +刻意放在 `skills/` 树**之外**，原因有三： + +- `scripts/build-classic-hosts.py` 会整棵同步 `.codebuddy/skills` 到 `.claude/` `.cursor/`， + 放树内会让每次改动都产生 `--check` drift，还会把你的单价复制进生成的宿主包； +- `config/pricing.json` 属于 skill 自带资产，后续更新会把它冲掉，覆盖文件不会； +- 它是纯数据文件，删除或改名即可停用，不需要重启任何东西（hook 每次都是新进程）。 + +**格式**：键是模型名（匹配时忽略大小写与首尾空格），值是要覆盖的字段。 +单位与 `config/pricing.json` 完全一致——**USD / 1M tokens**，可用字段只有四个： +`input`、`output`、`cache_read`、`cache_write`。 + +```json +{ + "my-model": { "output": 9.0 }, + "gpt-4o": { "output": 99.0 } +} +``` + +**合并是字段级的**：上面这份只改 `output`——`my-model` 的 `input`/`cache_read`/`cache_write` +沿用内置值（内置表里没有 `my-model` 时，未覆盖的字段按 0 计），`gpt-4o` 的 `input` 仍是内置的 `2.5`。 +可以新增内置表里不存在的模型，但**不能删除**内置的模型或字段（合并只能加不能减）。 + +**两个环境变量**（都支持 `~` 展开）： + +| 环境变量 | 作用 | 优先级 | +|---|---|---| +| `AOBS_PRICES_PATH` | **替换**整张基础表（不再读内置 `config/pricing.json`） | 低 | +| `AOBS_PRICING_OVERRIDES_PATH` | 在上面那张基础表之上**叠加**一份补丁，指向任意路径 | 高 | + +两者同时设置时，覆盖文件里的字段最终生效。 + +> ⚠️ **只对新事件生效，不回溯**：`cost_usd` 在 hook 期就写进了 `metrics.ndjson`， +> 补价之后只有**之后新产生**的 usage 事件按新价格计算，已经落盘的历史成本不会被重算。 + +**出问题会静默降级**：覆盖文件不存在、JSON 非法、顶层不是对象、字段值不是数字， +四种情况都自动退回纯内置价格表——不报错、不打印、hook 照常退出 0 +（否则一个手误的格式问题会让整条 hook 链路不可用）。所以"改了没生效"通常意味着文件没被读到， +先跑一次诊断： + +```bash +python3 - <<'PY' +import sys +sys.path.insert(0, "scripts") +from core import emitter +print("path :", emitter.resolve_overrides_path()) +print("loaded :", emitter.load_price_overrides()) +print("gpt-4o :", emitter.load_prices().get("gpt-4o")) +PY +``` + +`path` 是实际读取的位置（不是你以为的那个），`loaded` 为空说明文件没读到或全被判为坏字段。 + +**看板上的提示**：「建议关注」卡片第 4 条「模型定价」会列出**完全没命中价格表**的模型 +（模型名 + 缺少成本的 usage 事件数）。模糊匹配（最长子串）命中的模型能算出成本，不会出现在里面。 +处置方式就是把该模型写进覆盖文件；所有模型都有价时这条显示 green。 + +### 常见问题 + +- 没有日志：检查 hook 命令路径。 +- `ms` 为空：通常是 pre/post 未配对。 +- 没有 `usage`：`transcript_path` 缺失、tail 无 usage，或增量已去重。 +- AgentLens 未启用：检查 `.state.json` 的 `_agentlens.enabled` / `last_error`。 + +### 已知问题（未修复） + +- **工具失败事件的 `raw_response` 经常缺失（PostToolUse 与 transcript 落盘之间的时序竞争）**： + 真实 CodeBuddy CLI 场景下实测复现过——一次会故意制造失败的 `Bash` 调用（`exitCode=1`， + transcript 里 `function_call_result.providerData.toolResult.rawResponse` 确实带了 + `is_error:true`/`exitCode:1`/`tool_error_code`），但最终写进 `metrics.ndjson` 的 + `tool` 事件的 `tool_details` 里完全没有 `raw_response` 字段。 + 用实测时间戳定位到根因：PostToolUse hook 记录这次调用的时间是 `ts=...396.524`， + 但 transcript 里 `function_call_result` 真正落盘的时间是 `timestamp=...396.609`—— + **晚了 85ms**。hook 触发时去扫 transcript 文件的那一刻，CLI 还没来得及把执行结果 + 那条记录写盘，`find_current_tool_context()`（`core/collector.py`）只能看到 + `function_call`（请求），看不到 `function_call_result`（结果）。已经单独验证过 + `merge_tool_records()` / `tool_details_from_record()` 的合并逻辑本身没问题—— + 只要数据真的已经在文件里，能正确解析出完整 `raw_response`；问题纯粹是读的时机 + 比 CLI 写盘早了一步。`collect_transcript_entries()` 用的是持久化的增量字节 offset + 游标，错过这次窗口后不会在后续调用里回头补扫，所以这次机会永久丢失，直接后果是 + `build_dashboard_data.py` 的 `tool_call_failed()`（无论怎么改判定逻辑）都拿不到 + 数据，"工具失败率"/`failures` 列表对这类快速失败调用会漏检。 + 复现方式：让 CodeBuddy 执行一个必然快速失败的命令（如 `ls /path/does/not/exist`， + 越快的命令越容易复现，因为 hook 触发与 transcript 落盘之间的竞争窗口更紧张）， + 对比 `metrics.ndjson` 里该 `tool` 事件的 `tool_details.raw_response` 是否存在， + 和对应 transcript `.jsonl` 里 `function_call`/`function_call_result` 两条记录各自 + 的 `timestamp` 先后。 + + **影响范围（已精确定位，不是猜测）**：`raw_response` 在 `build_dashboard_data.py` + 里只有两处消费者——`tool_call_failed()`（喂给 `build_daily_and_sessions()` 的 + `day.failures` 计数和 `session.status`）和 `build_failures()`（"工具失败面板"的 + 数据源）。真实数据统计过：107 次 tool 事件里只有 2 次带 `raw_response`，且只有 + `Bash` 调用会带这个字段（Read/Edit/Write/SendMessage 等其它工具从不带，不受 + 此问题影响）。缺失时 `tool_call_failed()` 默认判"未失败"，所以效果是**恒定 + 漏报，不会误报**——工具失败率/失败面板/会话状态列显示的"正常"可能掩盖了真实 + 发生过的 Bash 失败。turns、duration、dispatch、cost、token、skill/rule 命中、 + devflow 阶段耗时、会话列表本身完全不受影响，是纯观测盲区，不影响 devflow 实际 + 执行行为。 + + **三次修复尝试均已失败，均已回滚（详见下方"已尝试且已放弃的修复方向"）**： + 真正阻塞方向 A 的证据很反常——三次独立测试里，预算从 160ms 加到 500ms 再加到 + 2000ms，实测缺口每次都精确地"比预算多几十到一百多毫秒"（206/564/2151ms）， + 不像是在等一个独立发生的固定延迟，更像是**hook 自己的同步等待在阻塞 CLI 落盘** + ——等得越久，结果来得越晚。这个因果关系还没验证清楚，在验证清楚之前，继续在 + hook 里加同步等待大概率是死路，不建议再尝试。 + +### 已尝试且已放弃的修复方向（供以后参考，避免重复踩坑） + +1. **方向 A：PostToolUse 里同步有界重试**（`claim=False` 轮询直到等到 `raw_response` + 或超时，只对 `tool_name=="Bash"` 生效）。三次真实端到端验证，预算 160ms/500ms/ + 2000ms 全部失败，且"缺口≈预算+常数"的规律强烈暗示等待本身可能在拖慢 CLI + 落盘（见上文）。不建议在搞清楚这层因果关系之前继续加大预算。 +2. **方向 B：推迟到 `pending_tool_emits` 重试队列，下次 hook 触发或 session Stop + 时再补**（`claim=False` 探测 + 延后 `claim=True`，避免过早消费掉 call_id）。 + 逻辑和单元测试都通过，但端到端验证暴露了一个更深的、独立于这次修复的既有 + 架构问题：`find_current_tool_context()`（工具上下文查询）和 + `emit_transcript_events()`（usage token 扫描）共用同一个持久化字节偏移游标 + （`core/state.py` 里只按 `(sid, transcript_path)` 区分，不分用途）。 + `emit_transcript_events()` 每次 `handle_post()` 都无条件推进这个游标，一旦 + 推过某段内容，后续任何工具上下文重试在这段范围内都会**彻底找不到任何数据** + （不只是缺 `raw_response`，连 `call_id`/`arguments` 都没了）——比不修复更糟。 + 真实验证过两次：两条端到端测试调用最终落盘时 `tool_details`完全是空的。 + 要让方向 B 真正可行，必须先给工具上下文查询一个独立于 usage 扫描的游标， + 这是范围更大、需要认真设计的改动，还没有细化方案。 + +以上两个方向的实现和回滚记录详见会话 memory（`project-agent-observability-raw-response-race`）。 diff --git a/.codebuddy/skills/agent-observability/references/schema-v2.md b/.codebuddy/skills/agent-observability/references/schema-v2.md new file mode 100644 index 0000000..9817940 --- /dev/null +++ b/.codebuddy/skills/agent-observability/references/schema-v2.md @@ -0,0 +1,282 @@ +## Schema v2.3 + +`agent-observability` 输出 `ndjson`，每行一个事件对象。 + +> 以下字段表基于当前 `logs/metrics.ndjson` 的实际输出整理；其中“可选字段”只会在特定场景出现。 + +### 通用字段 + +- `event`: `start | user_prompt_submit | tool | usage | stop | error | stage_transition` +- `sid`: session id +- `ts`: 秒级时间戳（float） +- `turn_id`: 可选；由 `UserPromptSubmit` 生成，贯穿本轮事件 + +### Devflow 感知字段（可选，见 `core/devflow.py`） + +只有当前项目正在跑 `.codebuddy/runtime` 描述的 multi-agents-devflow 工作流（存在 +`.codebuddy/teams/multi-agents-devflow-{task_slug}/` team 目录）时才会出现，非 +devflow 项目完全不受影响： + +- `tool` / `usage` / `stop` 事件上会附带 `task_slug`（devflow 需求标识，用于把 + main + 最多 7 个常驻角色跨事件串成"同一次运行"；⚠️ `/resume-devflow` 中断恢复 + 可能发生在新的 CodeBuddy 顶层 session 里，因此**聚合一次 devflow 运行要按 + `task_slug` 分组，不能按 `sid` 分组**）与 `stage`（`workflow-state.json` 的 + `current_stage`，如 `TASK-03`/`CODE-REVIEW`）。 +- `tool` 事件在能从 subagent transcript 文件名识别出并行 sub-developer 轨道时 + （`sub-developer-PT-01` 这类命名）会额外带 `pt_id`（如 `"PT-01"`）。按事件自己的 + transcript_path 推断，不依赖任何 session 级共享状态——并行轨道是真并发，不能 + 用一个可变指针记"当前是哪条轨道"。 + +### 事件字段 + +| event | 实际顶层字段 | 说明 | +|---|---|---| +| `start` | `event`, `sid`, `agent`, `ts` | 会话开始 | +| `user_prompt_submit` | `event`, `sid`, `agent`, `turn_id`, `prompt_len`, `ts` | 新 turn 边界；AgentLens 侧按 `1 Trace = 1 Turn` 生成 trace | +| `tool` | `event`, `sid`, `agent`, `tool`, `ms`, `transcript_path`, `turn_id`, `ts`, `skill`, `rule`, `cwd`, `call_id`, `message_id`, `tool_details`, `task_slug`?, `stage`?, `pt_id`? | 工具调用与耗时 | +| `usage` | `event`, `sid`, `agent`, `tool`, `tokens`, `model`, `transcript_path`, `source_offset`, `turn_id`, `message_id`, `ts`, `task_slug`?, `stage`? | transcript 增量 token / 模型 / 消息归属 | +| `stop` | `event`, `sid`, `agent`, `tokens`, `model`, `transcript_path`, `source_offset`, `turn_id`, `message_id`, `ts`, `task_slug`?, `stage`? | 会话尾部 flush | +| `error` | `event`, `sid`, `phase`, `error`, `x_traceback`, `ts` | hook 自身异常记录 | +| `stage_transition` | `event`, `sid`, `task_slug`, `stage`, `status`?, `executor`?, `retry_count`?, `review_result`?, `ts` | devflow `workflow-state.json` 某个 stage 的 status/retry_count/review_result 发生变化时触发（仅 devflow 项目） | + +### 可选字段说明 + +- `tool.ms`: Pre/Post 未成功配对时可能为 `null` +- `tool.call_id`: 仅当工具调用存在 call id 时出现 +- `tool.message_id`: 仅当成功和 transcript 中的 message 关联上时出现 +- `tool.tool_details`: 仅当 transcript 中能还原出更细工具上下文时出现 +- `usage.message_id`: 仅当 usage 对应的 transcript message 可识别时出现 +- `usage.model`: transcript 中能识别模型名时出现 +- `usage.cost_usd`: 只有模型价格命中价格表时才会出现。价格表 = 内置 `config/pricing.json` 与用户覆盖文件（默认 `/.codebuddy/agent-observability/pricing.overrides.json`，可用 `AOBS_PRICING_OVERRIDES_PATH` 指定）的字段级合并结果；覆盖只对该文件写入**之后**产生的新事件生效，不回溯重算历史成本（见 `quickstart.md` 的「自定义模型定价覆盖」） +- `stop.cost_usd`: 只有当前 stop 事件对应 usage 能估算成本时才会出现 +- `stop.cost_session_usd`: 只有 stop 汇总阶段能反算出整个 session 成本时才会出现 + +### 完整示例 + +换成更容易读的多行 JSON。下面仍然是当前 `metrics.ndjson` 里的实际数据，只是长文本字段做了截断。 + +`start` + +```json +{ + "event": "start", + "sid": "f6435d63-b4d5-44f2-b4d6-ae73d6c140fd", + "agent": "main", + "ts": 1784015331.87806 +} +``` + +`user_prompt_submit` + +```json +{ + "event": "user_prompt_submit", + "sid": "f6435d63-b4d5-44f2-b4d6-ae73d6c140fd", + "agent": "main", + "turn_id": "turn-1784015342075", + "prompt_len": 136, + "ts": 1784015342.076438 +} +``` + +`usage` + +```json +{ + "event": "usage", + "sid": "f6435d63-b4d5-44f2-b4d6-ae73d6c140fd", + "agent": "main", + "tool": "Read", + "tokens": { + "input": 31110, + "output": 594, + "cache_read": 3072, + "total": 31704 + }, + "model": "hy3-ioa", + "transcript_path": "/Users/rachel/.codebuddy/projects/Users-rachel-skillhub-tokentrack-mr/f6435d63-b4d5-44f2-b4d6-ae73d6c140fd.jsonl", + "source_offset": 17423, + "turn_id": "turn-1784015342075", + "message_id": "d187771c6eef4520a530564dd2a73e38", + "ts": 1784015355.413464 +} +``` + +`tool` 调用参数型 + +```json +{ + "event": "tool", + "sid": "f6435d63-b4d5-44f2-b4d6-ae73d6c140fd", + "agent": "main", + "tool": "Read", + "ms": null, + "transcript_path": "/Users/rachel/.codebuddy/projects/Users-rachel-skillhub-tokentrack-mr/f6435d63-b4d5-44f2-b4d6-ae73d6c140fd.jsonl", + "turn_id": "turn-1784015342075", + "ts": 1784015355.412617, + "skill": [], + "rule": [ + "global" + ], + "cwd": "/Users/rachel/skillhub-tokentrack-mr", + "call_id": "chatcmpl-tool-9e50f7ec1d7e4a27", + "message_id": "d187771c6eef4520a530564dd2a73e38", + "tool_details": { + "call_id": "chatcmpl-tool-9e50f7ec1d7e4a27", + "arguments": "{\"file_path\": \"/Users/rachel/skillhub-tokentrack-mr/assets/devflow.defaults.yaml\"}", + "arguments_display_text": "assets/devflow.defaults.yaml" + } +} +``` + +`tool` 返回结果型 + +```json +{ + "event": "tool", + "sid": "f6435d63-b4d5-44f2-b4d6-ae73d6c140fd", + "agent": "main", + "tool": "Bash", + "ms": 759, + "transcript_path": "/Users/rachel/.codebuddy/projects/Users-rachel-skillhub-tokentrack-mr/f6435d63-b4d5-44f2-b4d6-ae73d6c140fd.jsonl", + "turn_id": "turn-1784015342075", + "ts": 1784015356.071573, + "skill": [], + "rule": [ + "global" + ], + "cwd": "/Users/rachel/skillhub-tokentrack-mr", + "call_id": "chatcmpl-tool-898d45e0999b3e3c", + "message_id": "93b617ceebaa49458169b89b4ecae17b", + "tool_details": { + "call_id": "chatcmpl-tool-898d45e0999b3e3c", + "result_content": "Command: cd /Users/rachel/skillhub-tokentrack-mr && ls -la .codebuddy/teams/ ...", + "raw_response": { + "exitCode": 1, + "signal": null, + "interrupted": false, + "sandboxDenied": false, + "stderrBytesTruncated": 0, + "stdoutBytesTruncated": 0, + "tool_error_code": "8002", + "is_error": true, + "error": "Command: cd /Users/rachel/skillhub-tokentrack-mr && ls -la .codebuddy/teams/ ..." + }, + "output_text": "Command: cd /Users/rachel/skillhub-tokentrack-mr && ls -la .codebuddy/teams/ ...", + "original_message_id": "d187771c6eef4520a530564dd2a73e38", + "next_message_id": "93b617ceebaa49458169b89b4ecae17b", + "message_id_reassigned": true + } +} +``` + +`stop` + +```json +{ + "event": "stop", + "sid": "f6435d63-b4d5-44f2-b4d6-ae73d6c140fd", + "agent": "test-engineer", + "tokens": { + "input": 128290, + "output": 665, + "cache_read": 128192, + "total": 128955 + }, + "model": "hy3-ioa", + "transcript_path": "/Users/rachel/.codebuddy/projects/Users-rachel-skillhub-tokentrack-mr/f6435d63-b4d5-44f2-b4d6-ae73d6c140fd.jsonl", + "source_offset": 1157291, + "turn_id": "turn-1784015342075", + "message_id": "51e4fae9553d4d0995e7a599f3b2ef2d", + "ts": 1784018405.467215 +} +``` + +`error` + +```json +{ + "event": "error", + "sid": "6e8cd633-1a2f-4488-b1df-1eef168009c3", + "phase": "post", + "error": "AttributeError: module 'core.transcript_runtime' has no attribute 'transcript_path'", + "x_traceback": "Traceback ...", + "ts": 1784183163.4314518 +} +``` + +`stage_transition`（仅 devflow 项目；见 `core/devflow.py`） + +```json +{ + "event": "stage_transition", + "sid": "f6435d63-b4d5-44f2-b4d6-ae73d6c140fd", + "task_slug": "fix-token-bypass_20260911_0900", + "stage": "CODE-REVIEW", + "status": "failed", + "executor": "code-reviewer", + "retry_count": 1, + "review_result": "failed", + "ts": 1784015412.223 +} +``` + +### tokens + +```json +{"input": 1200, "output": 180, "cache_read": 9000, "cache_creation": 0, "total": 1380} +``` + +- `total`: 统一按 `input + output` 计算 +- `cache_read` / `cache_creation`: 保留给成本估算和缓存命中分析使用 + +### tool_details + +`tool` 事件里的 `tool_details` 是一个可选嵌套对象，当前实现里可能包含： + +- `call_id` +- `arguments` +- `arguments_display_text` +- `result_content` +- `raw_response` +- `output_text` +- `original_message_id` +- `next_message_id` +- `message_id_reassigned` + +常见示例： + +```json +{ + "call_id": "chatcmpl-tool-9e50f7ec1d7e4a27", + "arguments": "{\"file_path\": \"/path/to/file\"}", + "arguments_display_text": "path/to/file" +} +``` + +### AgentLens sidecar + +`.state.json[sid]._agentlens` 只作为内部状态使用，核心字段： + +- `enabled`: AgentLens 上报是否可用 +- `last_error`: 最近一次降级原因 +- `current_turn.carrier.traceparent`: 当前 turn 的 trace context +- `current_turn.subagent_spans`: 同一 turn 下的子 agent span carrier +- `current_turn.agent_spans`: 同一 turn 下按 agent 聚合的 span carrier 与累计统计 +- `current_turn.step_spans`: `(agent, message_id)` 级 step span registry +- `turn_history`: 已结束 turn 的摘要 + +### Hook 接入 + +参考 `templates/settings-hook.json`，必须启用： + +- `SessionStart` +- `UserPromptSubmit` +- `PreToolUse` +- `PostToolUse` +- `Stop` + +### 兼容 + +- 旧日志若在 `tool` 事件里直接携带 `tokens` / `cost_usd`，按 legacy usage 处理。 +- 不理解 `turn_id` 的下游消费者可安全忽略。 diff --git a/.codebuddy/skills/agent-observability/scripts/__init__.py b/.codebuddy/skills/agent-observability/scripts/__init__.py new file mode 100644 index 0000000..52073be --- /dev/null +++ b/.codebuddy/skills/agent-observability/scripts/__init__.py @@ -0,0 +1 @@ +"""agent-observability 的脚本包。""" diff --git a/.codebuddy/skills/agent-observability/scripts/build_dashboard_data.py b/.codebuddy/skills/agent-observability/scripts/build_dashboard_data.py new file mode 100644 index 0000000..e33f816 --- /dev/null +++ b/.codebuddy/skills/agent-observability/scripts/build_dashboard_data.py @@ -0,0 +1,624 @@ +#!/usr/bin/env python3 +"""把 `logs/metrics.ndjson` + `artifacts/*/workflow-state.json` + `.state.json` + +`hooks/logs/auto-dispatch.log` 聚合成看板需要的一份 `dashboard-data.json` 快照。 + +这是本地看板的数据源，不是 hook 链路的一部分——hook 只管往 metrics.ndjson 追加事件， +这个脚本单独、按需运行（比如每次想看一眼看板之前手动跑一次，或者配合文件监听器）。 +复用 core.devflow 的 stage_snapshot 做 workflow-state.json 的 schema 归一化， +不在这里重新实现一遍 Classic/Portable 的字段差异。 + +用法： + python3 build_dashboard_data.py --project-root <项目根目录> --out dashboard/dashboard-data.json + python3 build_dashboard_data.py --metrics-path ~/other/metrics.ndjson --state-path ~/other/.state.json + python3 build_dashboard_data.py --top-slow 5 + +不传 --metrics-path / --state-path 时，仍从 /logs/ 下的默认文件读取； +两者均支持 ~ 展开，输出 source 会如实反映实际读取路径。 + +--top-slow N 默认不开启，开启后只在终端额外打印耗时最长的 N 次 tool 事件， +不写入 dashboard-data.json（输出结构与不开时完全一致）。 + +没有任何真实数据时（hook 刚接上、还没跑过 session），会输出一份全空但结构合法的快照， +不会报错、也不会伪造数据。 +""" +from __future__ import annotations + +import argparse +import json +import sys +from collections import defaultdict +from pathlib import Path +from typing import Any + +SCRIPTS_DIR = Path(__file__).resolve().parent +if str(SCRIPTS_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPTS_DIR)) + +from core import devflow as dv # type: ignore + +try: # emitter 只在"未定价模型"统计里用到，导入失败只让这一项退化为 [] + from core import emitter as em # type: ignore +except Exception: # pragma: no cover - 目标机缺依赖时的兜底 + em = None # type: ignore + +# devflow 两套 schema 的 stage 名 -> 展示用 (label, name)。取不到的 stage 用 key 本身兜底。 +STAGE_META: dict[str, tuple[str, str]] = { + "PHASE-0": ("P0", "初始化 + 判定大小"), + "SOLO": ("SOLO", "单 agent 全流程"), + "TASK-01": ("T01", "需求分析 + 澄清"), + "REQUIREMENT": ("REQ", "需求分析 + 澄清"), + "TASK-02": ("T02", "技术方案"), + "DESIGN": ("DES", "技术方案"), + "TASK-03": ("T03", "代码实现"), + "IMPLEMENT": ("IMPL", "代码实现"), + "CODE-REVIEW": ("CR", "代码审查"), + "REVIEW": ("REV", "代码审查"), + "TASK-04": ("T04", "E2E 测试"), + "TEST": ("TEST", "E2E 测试"), + "TASK-05": ("T05", "知识沉淀"), + "KNOWLEDGE": ("KNOW", "知识沉淀"), + "SUMMARY": ("汇总", "最终汇总"), +} +# 两套 schema 各自的阶段顺序，用来按正确顺序渲染 stepper（不能直接遍历 dict，顺序不保证）。 +CLASSIC_ORDER = ["PHASE-0", "TASK-01", "TASK-02", "TASK-03", "CODE-REVIEW", "TASK-04", "TASK-05"] +# SOLO 完成后，small 任务会由 solo-developer 合并执行 TASK-05 知识沉淀（真实运行验证过， +# 不是理论上可选的分支）；stepper 顺序必须把它列进去，否则会把已完成的阶段悄悄漏掉。 +CLASSIC_SOLO_ORDER = ["PHASE-0", "SOLO", "TASK-05"] +PORTABLE_ORDER = ["PHASE-0", "REQUIREMENT", "DESIGN", "IMPLEMENT", "REVIEW", "TEST", "KNOWLEDGE", "SUMMARY"] +PORTABLE_SOLO_ORDER = ["PHASE-0", "SOLO", "SUMMARY"] + + +def read_ndjson(path: Path) -> list[dict[str, Any]]: + records: list[dict[str, Any]] = [] + if not path.is_file(): + return records + with path.open("r", encoding="utf-8") as fp: + for line in fp: + line = line.strip() + if not line.startswith("{"): + continue + try: + obj = json.loads(line) + except Exception: + continue + if isinstance(obj, dict): + records.append(obj) + return records + + +def safe_load_json(path: Path) -> Any: + try: + return json.loads(path.read_text("utf-8")) + except Exception: + return None + + +def day_of(ts: float) -> str: + from datetime import datetime, timezone + return datetime.fromtimestamp(ts, tz=timezone.utc).strftime("%Y-%m-%d") + + +def tool_call_failed(raw_response: Any) -> bool: + """判断一次工具调用是否失败。 + + 真实 CodeBuddy CLI 的 transcript（`function_call_result.providerData.toolResult. + rawResponse`）里从来没有 `is_error` 这个布尔字段——实际信号是 `exitCode`（非 0 + 即失败）和/或 `tool_error_code`（非 "0"/空即失败）。之前只认 `is_error`，导致 + 失败统计在这个宿主上永远是 0，不管命令是否真的失败（复合命令比如 + `cmd; echo ...` 会把失败进一步掩盖成 exitCode=0，那种情况下这里也如实判定为 + 未失败——判断整条工具调用本身有没有失败，不追究命令内部的子步骤）。 + 仍然保留 `is_error` 判断，兼容其它可能真的写这个字段的宿主。 + """ + if not isinstance(raw_response, dict): + return False + if raw_response.get("is_error"): + return True + exit_code = raw_response.get("exitCode") + if isinstance(exit_code, (int, float)) and exit_code != 0: + return True + tool_error_code = raw_response.get("tool_error_code") + if isinstance(tool_error_code, str) and tool_error_code.strip() not in ("", "0"): + return True + return False + + +def build_daily_and_sessions(events: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + by_day: dict[str, dict[str, Any]] = {} + by_sid: dict[str, dict[str, Any]] = {} + by_sid_turn: dict[tuple[str, str], dict[str, Any]] = {} + # AgentLens（写 state.current_turn 的那套 tracing）默认关闭时，tool/usage 事件 + # 自身的 turn_id 永远是 None——但 user_prompt_submit 事件不依赖 AgentLens， + # 始终携带真实 turn_id（见 emitter.build_prompt_submit_event）。按 ts 排序后， + # 把它当作 turn 边界，让后续没有自带 turn_id 的 tool/usage 事件归入"当前 sid + # 最近一次打开的 turn"，而不是全部塌缩进同一个 "?" 占位桶。 + current_turn_by_sid: dict[str, str] = {} + + def day_bucket(iso: str) -> dict[str, Any]: + return by_day.setdefault(iso, { + "iso": iso, "sessionCount": 0, "input": 0, "output": 0, "cache": 0, + "cost": 0.0, "toolCalls": 0, "failures": 0, "skillHits": defaultdict(int), + "_sids": set(), + }) + + def sess_bucket(sid: str) -> dict[str, Any]: + return by_sid.setdefault(sid, { + "id": sid, "date": None, "agents": set(), "turns": set(), "toolCalls": 0, + "tokens": 0, "cost": 0.0, "first_ts": None, "last_ts": None, "models": defaultdict(int), + "status": "ok", "timeline": {}, + }) + + for rec in sorted(events, key=lambda r: r.get("ts") if isinstance(r.get("ts"), (int, float)) else 0): + event = rec.get("event") + sid = str(rec.get("sid") or "") + ts = rec.get("ts") + if not sid or not isinstance(ts, (int, float)): + continue + iso = day_of(ts) + sess = sess_bucket(sid) + sess["_seen_day"] = iso + # 不管事件类型，只要这天有这个 sid 的任何事件就算一次"当日活跃会话"。 + # 之前只在 tool/usage 分支里 add，纯对话（只有 user_prompt_submit，没有 + # 触发任何工具调用也没有 usage）的会话永远不会被计入任何一天的 + # sessionCount——总览页的"会话数"会比"会话浏览"tab 里实际展开的会话数少 + # （真实数据复现过：5 个 session 只统计出 4 个 sessionCount）。 + day_bucket(iso)["_sids"].add(sid) + if sess["first_ts"] is None or ts < sess["first_ts"]: + sess["first_ts"] = ts + sess["date"] = iso + if sess["last_ts"] is None or ts > sess["last_ts"]: + sess["last_ts"] = ts + agent = rec.get("agent") + if agent: + sess["agents"].add(str(agent)) + raw_turn_id = rec.get("turn_id") + if event == "user_prompt_submit" and raw_turn_id: + current_turn_by_sid[sid] = str(raw_turn_id) + # 提前建桶——纯对话轮次（没有触发任何工具调用，只有 usage 都没有） + # 之前只在 tool/usage 分支里 setdefault，这种轮次永远不会出现在 + # timeline 里，导致 turns 计数和时间线展开的分组数对不上。 + by_sid_turn.setdefault( + (sid, str(raw_turn_id)), + {"turn": raw_turn_id, "agent": agent or "main", "events": []}, + ) + turn_id = raw_turn_id or current_turn_by_sid.get(sid) + if turn_id: + sess["turns"].add(str(turn_id)) + + if event == "tool": + day = day_bucket(iso) + day["toolCalls"] += 1 + sess["toolCalls"] += 1 + for s in rec.get("skill") or []: + day["skillHits"][str(s)] += 1 + tool_details = rec.get("tool_details") or {} + raw_response = tool_details.get("raw_response") if isinstance(tool_details, dict) else None + is_err = tool_call_failed(raw_response) + if is_err: + day["failures"] += 1 + sess["status"] = "error" + key = (sid, str(turn_id or "?")) + turn = by_sid_turn.setdefault(key, {"turn": turn_id or "?", "agent": agent or "main", "events": []}) + turn["events"].append({ + "kind": "tool", "tool": rec.get("tool"), + "ms": rec.get("ms") if isinstance(rec.get("ms"), (int, float)) else 0, + "err": is_err, + "agent": str(agent) if agent else "main", + }) + elif event == "usage": + day = day_bucket(iso) + tokens = rec.get("tokens") or {} + day["input"] += int(tokens.get("input") or 0) + day["output"] += int(tokens.get("output") or 0) + day["cache"] += int(tokens.get("cache_read") or 0) + cost = rec.get("cost_usd") + if isinstance(cost, (int, float)): + day["cost"] += float(cost) + sess["cost"] += float(cost) + total_tok = int(tokens.get("input") or 0) + int(tokens.get("output") or 0) + sess["tokens"] += total_tok + model = rec.get("model") + if model: + sess["models"][str(model)] += 1 + key = (sid, str(turn_id or "?")) + turn = by_sid_turn.setdefault(key, {"turn": turn_id or "?", "agent": agent or "main", "events": []}) + turn["events"].append({ + "kind": "usage", "tokens": total_tok, + "agent": str(agent) if agent else "main", + }) + elif event == "error": + sess["status"] = "error" + + for (sid, _turn_key), turn in by_sid_turn.items(): + sess = by_sid.setdefault(sid, sess_bucket(sid)) + sess["timeline"].setdefault(_turn_key, turn) + + daily = [] + for iso in sorted(by_day.keys()): + d = by_day[iso] + daily.append({ + "iso": iso, "sessionCount": len(d["_sids"]), "input": d["input"], "output": d["output"], + "cache": d["cache"], "cost": round(d["cost"], 4), "toolCalls": d["toolCalls"], + "failures": d["failures"], "skillHits": dict(d["skillHits"]), + }) + + sessions = [] + for sid, s in by_sid.items(): + if s["first_ts"] is None: + continue + duration = int((s["last_ts"] or s["first_ts"]) - s["first_ts"]) + model = max(s["models"].items(), key=lambda kv: kv[1])[0] if s["models"] else None + timeline_sorted = sorted(s["timeline"].values(), key=lambda t: str(t.get("turn") or "")) + for idx, t in enumerate(timeline_sorted, start=1): + t["turn"] = idx + sessions.append({ + "id": sid, "date": s["date"], "agent": ", ".join(sorted(s["agents"])) or "main", + "turns": len(s["turns"]) or len(timeline_sorted), "toolCalls": s["toolCalls"], + "tokens": s["tokens"], "cost": round(s["cost"], 4), "duration": duration, + "model": model, "status": s["status"], "timeline": timeline_sorted, + }) + sessions.sort(key=lambda s: s["date"] or "", reverse=True) + return daily, sessions + + +def build_model_costs(events: list[dict[str, Any]]) -> list[dict[str, Any]]: + totals: dict[str, float] = defaultdict(float) + for rec in events: + if rec.get("event") != "usage": + continue + model = rec.get("model") + cost = rec.get("cost_usd") + if model and isinstance(cost, (int, float)): + totals[str(model)] += float(cost) + rows = [{"name": name, "cost": round(cost, 4)} for name, cost in totals.items()] + rows.sort(key=lambda r: r["cost"], reverse=True) + return rows + + +def build_unpriced_models(events: list[dict[str, Any]]) -> list[dict[str, Any]]: + """聚合"完全没命中价格表"的模型（D4：`emitter.is_unpriced(model)` 为 True）。 + + 只**统计**不重算：`cost_usd` 在 hook 期就写进事件了，这里不会（也不该）回头 + 按新价格补算历史事件（D3：覆盖只对新事件生效）。判定入口复用 emitter， + 不在本文件里重写一套模型名匹配逻辑，避免与 hook 期的口径分叉。 + + 模型名缺失/为空的 usage 事件无法归因到某个模型，直接跳过不计数。 + """ + if em is None: + return [] + counts: dict[str, int] = defaultdict(int) + for rec in events: + if rec.get("event") != "usage": + continue + model = rec.get("model") + if not model or not str(model).strip(): + continue + name = str(model) + try: + if em.is_unpriced(name): + counts[name] += 1 + except Exception: + continue + rows = [{"name": name, "usageEvents": count} for name, count in counts.items()] + # 按 (-事件数, 名字) 排序：避免看板顺序抖动，两次生成的快照可直接逐字节比对。 + rows.sort(key=lambda r: (-r["usageEvents"], r["name"])) + return rows + + +def build_failures(events: list[dict[str, Any]]) -> list[dict[str, Any]]: + agg: dict[tuple[str, str], dict[str, Any]] = {} + for rec in events: + if rec.get("event") != "tool": + continue + tool_details = rec.get("tool_details") or {} + raw_response = tool_details.get("raw_response") if isinstance(tool_details, dict) else None + if not tool_call_failed(raw_response): + continue + tool = str(rec.get("tool") or "unknown") + code = str(raw_response.get("tool_error_code") or raw_response.get("exitCode") or "error") + key = (tool, code) + row = agg.setdefault(key, {"tool": tool, "code": code, "count": 0, "last": None, "_last_ts": 0.0}) + row["count"] += 1 + ts = rec.get("ts") + if isinstance(ts, (int, float)) and ts > row["_last_ts"]: + row["_last_ts"] = ts + row["last"] = day_of(ts) + rows = list(agg.values()) + for row in rows: + row.pop("_last_ts", None) + rows.sort(key=lambda r: r["count"], reverse=True) + return rows + + +def build_top_slow(events: list[dict[str, Any]], limit: int | None = None) -> list[dict[str, Any]]: + """取耗时最长的 N 次 tool 事件，按 ms 降序。 + + 只做"读"：不改动 events，也不参与 dashboard-data.json 的任何字段——调用方拿 + 结果去打印即可。`ms` 的兜底口径与 build_daily_and_sessions 保持一致（缺失或 + 非数字按 0 计），否则同一份日志会得出两个互相矛盾的工具耗时视图。 + """ + if not isinstance(limit, int) or limit <= 0: + return [] + rows: list[dict[str, Any]] = [] + for rec in events: + if rec.get("event") != "tool": + continue + rows.append({ + "tool": str(rec.get("tool") or "unknown"), + "ms": rec.get("ms") if isinstance(rec.get("ms"), (int, float)) else 0, + }) + rows.sort(key=lambda r: r["ms"], reverse=True) + return rows[:limit] + + +def format_top_slow(rows: list[dict[str, Any]]) -> str: + """把 top-slow 结果渲染成终端文本；没有可展示的行时返回空串（由调用方决定 + 是否打印，避免没有任何 tool 事件时空打一个表头）。""" + if not rows: + return "" + width = max(len(str(r["tool"])) for r in rows) + lines = [f"top-slow {len(rows)} tool calls (by ms):"] + for idx, row in enumerate(rows, start=1): + lines.append(f" {idx}. {str(row['tool']):<{width}} {row['ms']}ms") + return "\n".join(lines) + + +def build_skills(state: dict[str, Any]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + totals: dict[str, int] = defaultdict(int) + never_used: dict[str, dict[str, Any]] = {} + for sid, sess in state.items(): + if sid.startswith("_") or not isinstance(sess, dict): + continue + for name, rec in (sess.get("skills") or {}).items(): + if not isinstance(rec, dict): + continue + totals[name] += int(rec.get("count") or 0) + for name, rec in (sess.get("rules") or {}).items(): + if not isinstance(rec, dict) or int(rec.get("count") or 0) > 0: + continue + never_used[name] = {"name": name, "reason": "no_paths", "detail": "从未被路径推断或 active-rule 命中"} + for name, count in totals.items(): + if count == 0: + never_used.setdefault(name, {"name": name, "reason": "no_paths", "detail": "静态扫描到，但从未被任何工具调用命中"}) + skill_rows = [{"name": name, "count": count} for name, count in totals.items() if count > 0] + skill_rows.sort(key=lambda r: r["count"], reverse=True) + return skill_rows, sorted(never_used.values(), key=lambda r: r["name"]) + + + +# "team-lead" 是 CodeBuddy 原生 team 基础设施里 lead/orchestrator session 自己的 +# mailbox 名字，语义上就是 main（core/agent_identity.py::normalize_role_name 把它 +# 和 "main" 归为同一类）；不是一个真实存在的子 agent，不应出现在"派发目标"里。 +_DISPATCH_EXCLUDE_AGENTS = {"main", "team-lead"} + + +def build_dispatch(state: dict[str, Any]) -> list[dict[str, Any]]: + totals: dict[str, int] = defaultdict(int) + for sid, sess in state.items(): + if sid.startswith("_") or not isinstance(sess, dict): + continue + for entry in sess.get("agent_history") or []: + if not isinstance(entry, dict): + continue + evidence = str(entry.get("evidence") or "") + agent = entry.get("agent") + if not agent or agent in _DISPATCH_EXCLUDE_AGENTS: + continue + # 只统计真正代表"新派发"的证据：工具调用直接触发的 dispatch@， + # 或 inbox 扫描独立确认的 "main 派给了谁"（inbox@dispatch:*）。 + # 排除 inbox@report:*／inbox@handoff:*——那是子 agent 上报/移交， + # 不是 main 发起的新派发，计进来会把"派发次数"虚高（同一次真实 + # 派发经常先触发 dispatch@，随后又在 inbox 里被自己的上报回声一次）。 + is_dispatch = evidence.startswith("dispatch@") or evidence.startswith("inbox@dispatch:") + if is_dispatch: + totals[str(agent)] += 1 + rows = [{"agent": agent, "count": count} for agent, count in totals.items()] + rows.sort(key=lambda r: r["count"], reverse=True) + return rows + + +def build_auto_dispatch_stats(project_root: Path) -> dict[str, int]: + log_path = project_root / ".codebuddy" / "hooks" / "logs" / "auto-dispatch.log" + stats = {"auto_dispatch": 0, "fallback_to_main": 0, "passthrough": 0} + if not log_path.is_file(): + return stats + with log_path.open("r", encoding="utf-8") as fp: + for line in fp: + line = line.strip() + if not line.startswith("{"): + continue + try: + rec = json.loads(line) + except Exception: + continue + decision = str(rec.get("decision") or "") + if decision == "auto_dispatch": + stats["auto_dispatch"] += 1 + elif decision == "fallback_to_main": + stats["fallback_to_main"] += 1 + elif decision in {"passthrough"}: + stats["passthrough"] += 1 + return stats + + +def build_cost_by_task_slug(events: list[dict[str, Any]]) -> dict[str, float]: + """按 task_slug 汇总 usage/stop 事件的 cost_usd——workflow-state.json 自己不知道成本， + 这是唯一能把 devflow 运行和真实花费对上的地方（需要 emitter 已经把 task_slug 挂到事件上， + 见 core/runtime.py 的 devflow 接入）。""" + totals: dict[str, float] = defaultdict(float) + for rec in events: + if rec.get("event") not in {"usage", "stop"}: + continue + task_slug = rec.get("task_slug") + cost = rec.get("cost_usd") + if task_slug and isinstance(cost, (int, float)): + totals[str(task_slug)] += float(cost) + return dict(totals) + + +def build_devflow_runs(project_root: Path, cost_by_slug: dict[str, float] | None = None) -> list[dict[str, Any]]: + artifacts_root = project_root / "artifacts" + runs: list[dict[str, Any]] = [] + cost_by_slug = cost_by_slug or {} + if not artifacts_root.is_dir(): + return runs + for child in sorted(artifacts_root.iterdir()): + state_path = child / "workflow-state.json" + raw = dv.read_workflow_state(str(state_path)) + if not isinstance(raw, dict) or not isinstance(raw.get("stages"), dict): + continue + snap = dv.stage_snapshot(raw) + size_class = str(snap.get("size_class") or "medium") + # medium/large 的 workflow-state.json 也会带着 SOLO key（模板固定写入， + # 状态停在 "pending" 或 "skipped"，从未被真正执行）——不能只看 key 存不存在， + # 必须看 SOLO 阶段是不是真的跑过，否则每个 medium/large 任务都会被误判成 + # solo 路径，阶段视图漏掉 TASK-01/02/03/CODE-REVIEW/TASK-04。 + solo_info = snap["stages"].get("SOLO") + solo_status = str(solo_info.get("status") or "") if isinstance(solo_info, dict) else "" + solo_actually_ran = solo_status not in {"", "pending", "skipped"} + is_solo = size_class == "small" or (size_class not in {"small", "medium", "large"} and solo_actually_ran) + if snap.get("schema_version") == "2.0": + order = PORTABLE_SOLO_ORDER if is_solo else PORTABLE_ORDER + else: + order = CLASSIC_SOLO_ORDER if is_solo else CLASSIC_ORDER + stages_out = [] + total_cost = cost_by_slug.get(child.name, 0.0) + run_start = None + run_end = None + overall = "completed" + for key in order: + info = snap["stages"].get(key) + if info is None: + # PHASE-0 在两套 schema 里都不会出现在 stages{} 里（它是隐式完成的： + # workflow-state.json 一旦存在，就说明 Phase 0 已经跑完了）， + # 不能用"没有条目"直接兜底成 pending，那样会把已完成的阶段显示错。 + info = {"status": "completed", "retry_count": 0, "executor": None} if key == "PHASE-0" \ + else {"status": "pending", "retry_count": 0, "executor": None} + label, name = STAGE_META.get(key, (key[:4], key)) + status = str(info.get("status") or "pending") + retry_count = int(info.get("retry_count") or 0) + raw_stage = raw.get("stages", {}).get(key) if isinstance(raw.get("stages"), dict) else {} + duration = 0 + if isinstance(raw_stage, dict): + started = raw_stage.get("started_at") + completed = raw_stage.get("completed_at") + if isinstance(started, str) and isinstance(completed, str): + try: + from datetime import datetime + t0 = datetime.fromisoformat(started.replace("Z", "+00:00")) + t1 = datetime.fromisoformat(completed.replace("Z", "+00:00")) + duration = max(0, int((t1 - t0).total_seconds())) + if run_start is None or t0 < run_start: + run_start = t0 + if run_end is None or t1 > run_end: + run_end = t1 + except Exception: + duration = 0 + if status == "failed": + overall = "paused" + elif status == "in_progress" and overall != "paused": + overall = "running" + stages_out.append({ + "key": key, "label": label, "name": name, + "exec": info.get("executor") or "-", "status": status, + "retry_count": retry_count, "duration": duration, + }) + # 按真实起止时间跨度算总时长（而不是逐阶段 duration 相加）——阶段之间可能 + # 有重叠（比如 TASK-05 的 started_at 早于 SOLO 的 completed_at），相加会 + # 把重叠部分重复计入，虚高于源数据本身反映的运行时长。 + total_duration = int((run_end - run_start).total_seconds()) if run_start and run_end else 0 + runs.append({ + "slug": child.name, "size": size_class, "stages": stages_out, + "cost": round(total_cost, 4), "duration": total_duration, "overall": overall, + }) + runs.sort(key=lambda r: r["slug"], reverse=True) + return runs + + +def resolve_input_paths( + skill_root: Path, + metrics_path: str | None = None, + state_path: str | None = None, +) -> tuple[Path, Path]: + """解析 metrics.ndjson 与 .state.json 的实际读取路径。 + + 任一参数为 None 时回退到 ``/logs/`` 下的默认路径，保证未传参时 + 行为与旧版本完全一致（向后兼容，不会破坏现有看板数据源）。 + + 非 None 时按用户给定路径（支持 ``~`` 展开）取绝对路径，便于看板从非默认 + 位置（如其它会话/项目的日志目录）聚合数据。 + """ + default_metrics = skill_root / "logs" / "metrics.ndjson" + default_state = skill_root / "logs" / ".state.json" + if metrics_path: + metrics = Path(metrics_path).expanduser().absolute() + else: + metrics = default_metrics + if state_path: + state = Path(state_path).expanduser().absolute() + else: + state = default_state + return metrics, state + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--project-root", default=".", help="devflow 项目根目录（含 .codebuddy/ 和 artifacts/）") + parser.add_argument("--out", default=None, help="输出路径，默认 /scripts/dashboard/dashboard-data.json") + parser.add_argument("--metrics-path", default=None, help="覆盖 metrics.ndjson 的读取路径（默认 /logs/metrics.ndjson，支持 ~ 展开）") + parser.add_argument("--state-path", default=None, help="覆盖 .state.json 的读取路径（默认 /logs/.state.json，支持 ~ 展开）") + parser.add_argument("--top-slow", type=int, default=None, help="额外在终端打印耗时最长的 N 次 tool 事件（工具名 + 耗时 ms），默认不打印；只打印不写入 dashboard-data.json") + args = parser.parse_args() + + project_root = Path(args.project_root).expanduser().resolve() + skill_root = SCRIPTS_DIR.parent + log_path, state_path = resolve_input_paths(skill_root, args.metrics_path, args.state_path) + + # 价格表带 lru_cache：本脚本可能长期驻留（配合文件监听器），单测也会同进程多次 + # 调 main()，每次都先清一次缓存，保证覆盖文件的改动当场生效。 + if em is not None: + try: + em.clear_price_cache() + except Exception: + pass + + events = read_ndjson(log_path) + state = safe_load_json(state_path) or {} + if not isinstance(state, dict): + state = {} + + daily, sessions = build_daily_and_sessions(events) + model_costs = build_model_costs(events) + unpriced = build_unpriced_models(events) + failures = build_failures(events) + skills, never_used = build_skills(state) + dispatch = build_dispatch(state) + auto_dispatch_stats = build_auto_dispatch_stats(project_root) + cost_by_slug = build_cost_by_task_slug(events) + devflow_runs = build_devflow_runs(project_root, cost_by_slug) + + from datetime import datetime, timezone + out_data = { + "generated_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "source": { + "metrics_ndjson": str(log_path), "state_json": str(state_path), + "project_root": str(project_root), "event_count": len(events), + }, + "daily": daily, "sessions": sessions, "modelCosts": model_costs, "unpricedModels": unpriced, + "failures": failures, + "skills": skills, "neverUsed": never_used, "dispatch": dispatch, + "autoDispatchStats": auto_dispatch_stats, "devflowRuns": devflow_runs, + } + + out_path = Path(args.out).expanduser().resolve() if args.out else (SCRIPTS_DIR / "dashboard" / "dashboard-data.json") + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(json.dumps(out_data, ensure_ascii=False, indent=2), encoding="utf-8") + print(f"wrote {out_path} ({len(events)} events, {len(sessions)} sessions, {len(devflow_runs)} devflow runs)") + # top-slow 放在原输出之后：先保证不传参时的终端输出与 JSON 行为完全不变。 + top_slow_block = format_top_slow(build_top_slow(events, args.top_slow)) + if top_slow_block: + print(top_slow_block) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.codebuddy/skills/agent-observability/scripts/core/__init__.py b/.codebuddy/skills/agent-observability/scripts/core/__init__.py new file mode 100644 index 0000000..1d6ad5e --- /dev/null +++ b/.codebuddy/skills/agent-observability/scripts/core/__init__.py @@ -0,0 +1,8 @@ +"""agent-observability 的核心运行时模块集合。 + +这里放的是 hook 主流程真正依赖的内部实现： +- 采集与 transcript 解析 +- 状态读写与去重 +- skill/rule 扫描 +- AgentLens trace 镜像 +""" diff --git a/.codebuddy/skills/agent-observability/scripts/core/agent_identity.py b/.codebuddy/skills/agent-observability/scripts/core/agent_identity.py new file mode 100644 index 0000000..270ff90 --- /dev/null +++ b/.codebuddy/skills/agent-observability/scripts/core/agent_identity.py @@ -0,0 +1,587 @@ +"""Agent 身份解析层。 + +这一层负责统一解析“当前是谁在工作、又把任务派给了谁”： +- 从 tool 调用参数里提取 agent 线索 +- 从 team inbox / mailbox 中补偿推断 agent 身份 +- 把解析结果写回 session state，供后续日志归因复用 +""" +from __future__ import annotations + +import json +import os +import re +import time +from datetime import datetime, timezone +from functools import lru_cache +from pathlib import Path +from typing import Any, Optional + +from . import scanner, state as st + +AGENT_PATH_RE = re.compile(r"\.codebuddy/agents/([a-zA-Z0-9_\-]+)(?:\.md)?", re.I) +DISPATCH_TOOLS = {"Task", "task", "Agent", "DeferExecuteTool"} +MESSAGE_TOOLS = {"send_message", "SendMessage"} +TEAM_PREFIX = "multi-agents-devflow-" +ROLE_INSTANCE_RE = re.compile(r"^(?P[a-z0-9][a-z0-9\-]*?)-\d+$", re.I) +INITIAL_ASSIGNMENT_RE = re.compile( + r"Initial task assignment for (?P[a-z0-9][a-z0-9\-]*)", + re.I, +) +ROLE_DECLARATION_RE = re.compile( + r"角色[：:]\s*(?P[a-z0-9][a-z0-9\-]*)", + re.I, +) +# devflow TASK-03 并行 fan-out 时，developer 会以 Task(name="sub-developer-PT-01", ...) +# 派发多条并行轨道。轨道号只在 transcript 文件名（= Task 的 name 参数）里出现， +# 按 subagent_name 归一化后会被折成同一个 "developer" —— 这是有意的（用于按角色汇总）， +# 轨道号需要单独提取，见 pt_id_from_transcript_path。 +PT_TRACK_RE = re.compile(r"-((?:PT|pt)-\d+)$") + + +def normalize_role_name(name: str | None) -> Optional[str]: + """把角色名归一化成稳定标识，例如把实例名折叠回基础角色名。""" + if not isinstance(name, str): + return None + cand = name.strip().lower() + if not cand: + return None + if cand.endswith(".json"): + cand = cand[:-5] + if "@" in cand: + cand = cand.split("@", 1)[0] + if cand in {"team-lead", "main"}: + return "main" + match = ROLE_INSTANCE_RE.match(cand) + if match: + cand = match.group("base") + return cand or None + + +def resolve_teams_root() -> Path: + """解析 CodeBuddy team inbox 的根目录。""" + config_dir = (os.environ.get("CODEBUDDY_CONFIG_DIR") or "").strip() + if config_dir: + return Path(config_dir).expanduser() / "teams" + return Path.home() / ".codebuddy" / "teams" + + +def resolve_team_dir(cwd: str, sid: str, cached_team_dir: str | None = None) -> Path | None: + """根据 session id 和 cwd 找到当前会话对应的 team 目录。""" + if cached_team_dir: + cached = Path(cached_team_dir).expanduser() + if (cached / "config.json").is_file(): + cfg = _load_json(cached / "config.json") + if isinstance(cfg, dict) and sid and str(cfg.get("leadSessionId") or "") == sid: + return cached + + if not sid: + return None + + teams_root = resolve_teams_root() + if not teams_root.is_dir(): + return None + + best_dir: Path | None = None + best_created = -1 + for team_dir in teams_root.iterdir(): + if not team_dir.is_dir() or not team_dir.name.startswith(TEAM_PREFIX): + continue + config = _load_json(team_dir / "config.json") + if not isinstance(config, dict): + continue + if str(config.get("leadSessionId") or "") != sid: + continue + if not _config_matches_cwd(config, cwd): + continue + + created_at = _safe_int(config.get("createdAt")) + if created_at <= 0: + try: + created_at = int(team_dir.stat().st_mtime * 1000) + except Exception: + created_at = 0 + if created_at > best_created: + best_dir = team_dir + best_created = created_at + return best_dir + + +def read_recent_messages( + team_dir: Path, + offsets: dict[str, int] | None = None, + *, + max_per_mailbox: int = 30, +) -> tuple[list[dict[str, Any]], dict[str, int]]: + """按 mailbox 增量读取最近消息，并返回新的 offset 游标。""" + inbox_dir = team_dir / "inboxes" + if not inbox_dir.is_dir(): + return [], offsets or {} + + prev_offsets = offsets or {} + new_offsets: dict[str, int] = {} + messages: list[dict[str, Any]] = [] + + for inbox_file in sorted(inbox_dir.glob("*.json")): + raw = _load_json(inbox_file) + if not isinstance(raw, list): + raw = [] + + total = len(raw) + prev = _safe_int(prev_offsets.get(inbox_file.name)) + if prev < 0 or prev > total: + prev = 0 + + start = prev + if start == 0 and total > max_per_mailbox: + start = total - max_per_mailbox + + mailbox_name = inbox_file.stem + mailbox_role = normalize_role_name(mailbox_name) + for idx, item in enumerate(raw[start:], start=start): + norm = _normalize_message(item, mailbox_name, mailbox_role, idx) + if norm: + messages.append(norm) + + new_offsets[inbox_file.name] = total + + messages.sort(key=lambda x: (x.get("ts") or 0.0, x.get("mailbox_name") or "", x.get("index") or 0)) + return messages, new_offsets + + +def infer_identity_from_messages(messages: list[dict[str, Any]]) -> tuple[Optional[str], Optional[str], dict[str, Any]]: + """根据 inbox 消息流推断当前 agent 与派发目标。""" + current: Optional[str] = None + dispatched: Optional[str] = None + evidence: Optional[str] = None + + for msg in messages: + mailbox_name = str(msg.get("mailbox_name") or "") + mailbox_role = normalize_role_name(msg.get("mailbox_role")) + from_role = normalize_role_name(msg.get("from_role")) + payload = msg.get("payload") if isinstance(msg.get("payload"), dict) else {} + next_target = _extract_next_target(payload) + + if from_role == "main" and mailbox_role and mailbox_role != "main": + if not msg.get("is_shutdown"): + current = mailbox_role + dispatched = mailbox_role + evidence = f"dispatch:{mailbox_name}" + continue + + if mailbox_name == "team-lead" and from_role and from_role not in {"main", "system"}: + report_role = normalize_role_name( + payload.get("from_role") if isinstance(payload, dict) else None + ) or from_role + if next_target: + current = next_target + dispatched = next_target + evidence = f"handoff:{report_role}->{next_target}" + else: + current = report_role + evidence = f"report:{report_role}" + + meta: dict[str, Any] = {"messages_seen": len(messages)} + if evidence: + meta["evidence"] = evidence + return current, dispatched, meta + + +def safe_agent_from_provider(record: dict[str, Any]) -> str | None: + """当 transcript 自己带有 providerData 时，直接读取其中的 agent。""" + provider = record.get("providerData") + if not isinstance(provider, dict): + return None + agent = provider.get("agent") + if isinstance(agent, str) and agent.strip(): + return normalize_role_name(agent.strip()) + return None + + +def role_from_content_items(content: Any) -> str | None: + """从 transcript content 文本中提取更具体的业务角色名。""" + if not isinstance(content, list): + return None + for item in content: + if not isinstance(item, dict): + continue + text = item.get("text") + if not isinstance(text, str): + continue + for regex in (INITIAL_ASSIGNMENT_RE, ROLE_DECLARATION_RE): + match = regex.search(text) + if not match: + continue + role = normalize_role_name(match.group("role")) + if role: + return role + return None + + +@lru_cache(maxsize=256) +def role_for_subagent_transcript(transcript_path: str) -> str | None: + """从 subagent transcript 前几行推断更准确的角色名。""" + path = Path(transcript_path) + try: + with path.open("r", encoding="utf-8") as fp: + for _ in range(6): + line = fp.readline() + if not line: + break + try: + record = json.loads(line) + except Exception: + continue + + # 对 subagent transcript，优先信任务分配文本里的具体角色， + # 再回退到 providerData.agent，避免 general-purpose 覆盖业务角色。 + content_role = role_from_content_items(record.get("content")) + if content_role: + return content_role + + provider_agent = safe_agent_from_provider(record) + if provider_agent: + return provider_agent + except Exception: + return None + return None + + +def agent_for_transcript_path(transcript_path: str, fallback: str | None = None) -> str: + """把 transcript 路径映射回实际对应的 agent 身份。""" + parts = Path(transcript_path).parts + if "subagents" in parts: + return role_for_subagent_transcript(transcript_path) or Path(transcript_path).stem + return fallback or "main" + + +def pt_id_from_transcript_path(transcript_path: str) -> str | None: + """从并行 sub-developer 的 transcript 文件名里提取 PT 轨道号（如 "PT-01"）。 + + 按事件自己的 transcript_path 推断，而不是写一个共享的 session 级"当前 PT"指针—— + devflow 一次可以并行派发最多 6 条 sub-developer 轨道，它们在同一个 sid 下 + 真·并发运行，任何"当前是哪条轨道"的可变共享状态在并发场景下都是错的。 + """ + parts = Path(transcript_path).parts + if "subagents" not in parts: + return None + match = PT_TRACK_RE.search(Path(transcript_path).stem) + return match.group(1).upper() if match else None + + +def merge_agent_identity_from_inbox( + state_path: Path, + sid: str, + cwd: str, + active_agent: str | None, + dispatched: str | None, +) -> tuple[str, str | None]: + """在基于 tool 的推断之上，再叠加 mailbox/inbox 的证据。""" + + def _update(state: dict[str, Any]) -> tuple[str, str | None]: + sess = st.ensure_session(state, sid) + cached_team_dir = str(sess.get("_team_dir") or "").strip() or None + offsets = sess.get("_inbox_offsets") + if not isinstance(offsets, dict): + offsets = {} + + effective_agent = active_agent or str(sess.get("current_agent") or "main") + effective_dispatched = dispatched + team_dir = resolve_team_dir(cwd, sid, cached_team_dir) + if team_dir is None: + return effective_agent, effective_dispatched + + sess["_team_dir"] = str(team_dir) + messages, new_offsets = read_recent_messages(team_dir, offsets) + sess["_inbox_offsets"] = new_offsets + inferred_current, inferred_dispatched, meta = infer_identity_from_messages(messages) + if inferred_current: + effective_agent = inferred_current + if inferred_dispatched: + effective_dispatched = inferred_dispatched + if inferred_current or inferred_dispatched: + target_agent = inferred_current or inferred_dispatched + if target_agent: + sess["current_agent"] = target_agent + hist = sess.setdefault("agent_history", []) + hist.append({ + "ts": time.time(), + "agent": target_agent, + "evidence": f"inbox@{meta.get('evidence') or 'inbox'}", + }) + return effective_agent, effective_dispatched + + return st.update_state_locked(state_path, _update) + + +def resolve_active_agent_for_event( + *, + state_path: Path, + sid: str, + cwd: str, + data: dict[str, Any], +) -> tuple[str, str | None]: + """优先用 tool 线索、其次用 inbox 线索，解析事件对应的 agent。""" + tracker = AgentIdentityResolver() + active_agent, dispatched = tracker.track(state_path, sid, data) + if scanner.is_brainstorming_call(data): + + def _update(state_data: dict[str, Any]) -> None: + sess = st.ensure_session(state_data, sid) + sess["current_agent"] = "main" + + st.update_state_locked(state_path, _update) + return "main", dispatched + return merge_agent_identity_from_inbox( + state_path, + sid, + cwd, + active_agent, + dispatched, + ) + + +class AgentIdentityResolver: + """基于 tool 调用内容做一轮 agent 身份推断。""" + def __init__(self, known_agents: set[str] | None = None): + self.known_agents = known_agents or set() + + def extract_identity(self, data: dict) -> tuple[Optional[str], Optional[str]]: + tool = data.get("tool_name", "") + tool_input = data.get("tool_input") or {} + if not isinstance(tool_input, dict): + tool_input = {} + + current: Optional[str] = None + dispatched: Optional[str] = None + + if tool in MESSAGE_TOOLS: + extracted_current, extracted_dispatched = self._extract_from_message(tool_input) + if extracted_current and not current: + current = extracted_current + if extracted_dispatched and not dispatched: + dispatched = extracted_dispatched + + if tool in DISPATCH_TOOLS and not dispatched: + dispatched = self._extract_from_task(tool_input) + + if not current: + current = self._extract_from_path(data, tool_input) + + if self.known_agents: + if current and current not in self.known_agents: + current = None + if dispatched and dispatched not in self.known_agents: + dispatched = None + + return current, dispatched + + def track(self, state_path: Path, sid: str, data: dict) -> tuple[str, Optional[str]]: + tool = data.get("tool_name", "") + cur_agent, dispatched = self.extract_identity(data) + + def _update(state: dict[str, Any]) -> tuple[str, Optional[str]]: + sess = st.ensure_session(state, sid) + hist = sess.setdefault("agent_history", []) + prev_agent = sess.get("current_agent") or "main" + + if cur_agent and cur_agent != prev_agent: + sess["current_agent"] = cur_agent + hist.append({"ts": time.time(), "agent": cur_agent, "evidence": f"from_role@{tool}"}) + active_agent = cur_agent + else: + active_agent = prev_agent + + if dispatched and dispatched != active_agent: + dis = sess.setdefault("dispatched", {}) + if not isinstance(dis, dict): + dis = {} + sess["dispatched"] = dis + dis[dispatched] = dis.get(dispatched, 0) + 1 + sess["current_agent"] = dispatched + hist.append({"ts": time.time(), "agent": dispatched, "evidence": f"dispatch@{tool}<-{active_agent}"}) + + if len(hist) > 100: + sess["agent_history"] = hist[-100:] + + return active_agent, dispatched + + return st.update_state_locked(state_path, _update) + + def _extract_from_message(self, tool_input: dict) -> tuple[Optional[str], Optional[str]]: + current: Optional[str] = None + dispatched: Optional[str] = None + content = tool_input.get("content") + + if isinstance(content, str) and content.strip().startswith("{"): + try: + payload = json.loads(content) + current = self._extract_str(payload, "from_role") + next_target = payload.get("next_target") or {} + if isinstance(next_target, dict): + dispatched = ( + self._extract_str(next_target, "role_name") + or self._extract_str(next_target, "subagent_name") + ) + except Exception: + pass + elif isinstance(content, dict): + current = self._extract_str(content, "from_role") + next_target = content.get("next_target") or {} + if isinstance(next_target, dict): + dispatched = ( + self._extract_str(next_target, "role_name") + or self._extract_str(next_target, "subagent_name") + ) + + if not dispatched: + recipient = tool_input.get("recipient") + if isinstance(recipient, str): + normalized = recipient.strip().lower() + if normalized and normalized != "main": + dispatched = normalized + + return current, dispatched + + def _extract_from_task(self, tool_input: dict) -> Optional[str]: + sub = ( + # Claude Code 的 Agent 工具（非 CodeBuddy 原生 Task/team_create）用的是 + # subagent_type 字段，不是 subagent_name——不认这个字段会导致 dispatch + # 统计漏掉所有走 Agent 工具派发的场景。 + tool_input.get("subagent_type") + or tool_input.get("subagent_name") + or tool_input.get("name") + or tool_input.get("agent") + or tool_input.get("role") + ) + if isinstance(sub, str) and sub.strip(): + candidate = sub.strip().lower() + if "/" in candidate: + candidate = candidate.rsplit("/", 1)[1].replace(".md", "") + return candidate + return None + + def _extract_from_path(self, data: dict, tool_input: dict) -> Optional[str]: + text = json.dumps(tool_input, ensure_ascii=False) + " " + (data.get("cwd") or "") + match = AGENT_PATH_RE.search(text) + if match: + return match.group(1).lower() + return None + + @staticmethod + def _extract_str(payload: dict, key: str) -> Optional[str]: + value = payload.get(key) + if isinstance(value, str) and value.strip(): + return value.strip().lower() + return None + + +def _config_matches_cwd(config: dict[str, Any], cwd: str) -> bool: + """判断 team 配置中的成员 cwd 是否覆盖当前工作目录。""" + members = config.get("members") + if not isinstance(members, list): + return False + for member in members: + if not isinstance(member, dict): + continue + member_cwd = member.get("cwd") + if isinstance(member_cwd, str) and _paths_equal(member_cwd, cwd): + return True + return False + + +def _paths_equal(a: str, b: str) -> bool: + """按归一化绝对路径语义比较两个路径是否相等。""" + try: + return Path(a).expanduser().resolve() == Path(b).expanduser().resolve() + except Exception: + return os.path.abspath(os.path.expanduser(a)) == os.path.abspath(os.path.expanduser(b)) + + +def _normalize_message(item: Any, mailbox_name: str, mailbox_role: str | None, index: int) -> dict[str, Any] | None: + """把原始 inbox 消息归一化成统一可推断的结构。""" + if not isinstance(item, dict): + return None + + text = item.get("text") + payload = _coerce_payload(text) + summary = item.get("summary") if isinstance(item.get("summary"), str) else "" + from_raw = item.get("from") if isinstance(item.get("from"), str) else None + from_role = normalize_role_name((payload.get("from") if isinstance(payload, dict) else None) or from_raw) + ts = _parse_timestamp(item.get("timestamp")) + + return { + "mailbox_name": mailbox_name, + "mailbox_role": mailbox_role, + "from": from_raw, + "from_role": from_role, + "text": text if isinstance(text, str) else "", + "summary": summary, + "payload": payload, + "timestamp": item.get("timestamp"), + "ts": ts, + "index": index, + "is_shutdown": _is_shutdown_message(payload, summary, text), + } + + +def _extract_next_target(payload: dict[str, Any] | None) -> Optional[str]: + """从 payload 中提取下一跳要派发给的角色。""" + if not isinstance(payload, dict): + return None + next_target = payload.get("next_target") or {} + if not isinstance(next_target, dict): + return None + return normalize_role_name(next_target.get("role_name") or next_target.get("subagent_name")) + + +def _coerce_payload(text: Any) -> dict[str, Any] | None: + """把文本内容尽量解析成 JSON 字典。""" + if isinstance(text, dict): + return text + if not isinstance(text, str): + return None + stripped = text.strip() + if not stripped.startswith("{"): + return None + try: + data = json.loads(stripped) + if isinstance(data, dict): + return data + except Exception: + return None + return None + + +def _is_shutdown_message(payload: dict[str, Any] | None, summary: str, text: Any) -> bool: + """识别一条 inbox 消息是否表示 agent 关闭或退出。""" + if isinstance(payload, dict) and str(payload.get("type") or "").lower() == "shutdown_request": + return True + summary_low = summary.lower() if isinstance(summary, str) else "" + text_low = text.lower() if isinstance(text, str) else "" + return "shutdown" in summary_low or "shutdown request" in text_low + + +def _parse_timestamp(value: Any) -> float: + """把消息时间字段解析成 Unix 时间戳。""" + if not isinstance(value, str) or not value.strip(): + return datetime.now(timezone.utc).timestamp() + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp() + except Exception: + return datetime.now(timezone.utc).timestamp() + + +def _load_json(path: Path) -> Any: + """安全读取 JSON 文件，失败时返回 ``None``。""" + try: + return json.loads(path.read_text("utf-8")) + except Exception: + return None + + +def _safe_int(value: Any) -> int: + """把任意值尽量转成整数，失败时返回 0。""" + try: + return int(value or 0) + except Exception: + return 0 diff --git a/.codebuddy/skills/agent-observability/scripts/core/agentlens/__init__.py b/.codebuddy/skills/agent-observability/scripts/core/agentlens/__init__.py new file mode 100644 index 0000000..2fea21d --- /dev/null +++ b/.codebuddy/skills/agent-observability/scripts/core/agentlens/__init__.py @@ -0,0 +1,9 @@ +"""可选的 AgentLens sink，用于实时镜像 trace。""" +from .runtime import emit_post_step, emit_session_start, emit_session_stop, emit_turn_start + +__all__ = [ + "emit_post_step", + "emit_session_start", + "emit_session_stop", + "emit_turn_start", +] diff --git a/.codebuddy/skills/agent-observability/scripts/core/agentlens/bootstrap.py b/.codebuddy/skills/agent-observability/scripts/core/agentlens/bootstrap.py new file mode 100644 index 0000000..d6fc970 --- /dev/null +++ b/.codebuddy/skills/agent-observability/scripts/core/agentlens/bootstrap.py @@ -0,0 +1,229 @@ +from __future__ import annotations +"""AgentLens 启动辅助层。 + +这一层负责“读入与初始化”： +- 解析项目配置与环境变量 +- 延迟加载可选的 zhiyan 运行时 +- 缓存初始化结果，避免重复 init +- 处理调试落盘与失败状态写回 +""" + +import getpass +import json +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from .. import state as st + +DEFAULT_APP_NAME = "skillhub.codebuddy-hooks" +DEFAULT_BUSINESS_SCENARIO = "codebuddy-hook" + +_RUNTIME: dict[str, Any] | None = None +_INIT_SIGNATURE: tuple[str, str, str] | None = None + + +@dataclass(frozen=True) +class AgentLensConfig: + """从环境变量与 env 文件归一化得到的 AgentLens 配置。""" + endpoint: str | None + api_key: str | None + app_name: str + business_scenario: str + user: str + + @property + def enabled(self) -> bool: + return bool(self.endpoint and self.api_key) + + +def project_root(cwd: str | None = None) -> Path: + """解析用于查找 `.env` 的项目根目录。""" + if cwd: + return Path(cwd).expanduser().resolve() + env_cwd = str(os.environ.get("CODEBUDDY_PROJECT_DIR") or "").strip() + if env_cwd: + return Path(env_cwd).expanduser().resolve() + return Path.cwd().resolve() + + +def read_env_file(path: Path) -> dict[str, str]: + """以轻量方式解析 shell 风格的 env 文件，不做 source。""" + values: dict[str, str] = {} + if not path.is_file(): + return values + try: + lines = path.read_text("utf-8").splitlines() + except Exception: + return values + + for raw_line in lines: + line = raw_line.strip() + if not line or line.startswith("#"): + continue + if line.startswith("export "): + line = line[7:].strip() + if "=" not in line: + continue + key, value = line.split("=", 1) + key = key.strip() + value = value.strip() + if not key: + continue + if value and value[0] == value[-1] and value[0] in {"'", '"'}: + value = value[1:-1] + values[key] = value + return values + + +def load_config(project_root_path: Path) -> AgentLensConfig: + """按优先级加载配置：进程环境变量 > `.env.local` > `.env`。""" + env_file = read_env_file(project_root_path / ".env") + env_local_file = read_env_file(project_root_path / ".env.local") + + def _value(name: str, default: str | None = None) -> str | None: + env_value = os.environ.get(name) + if env_value is not None and str(env_value).strip(): + return str(env_value).strip() + if name in env_local_file and str(env_local_file[name]).strip(): + return str(env_local_file[name]).strip() + if name in env_file and str(env_file[name]).strip(): + return str(env_file[name]).strip() + return default + + user = _value("ZHIYANLLM_USER") + if not user: + user = ( + str(os.environ.get("USER") or "").strip() + or str(os.environ.get("USERNAME") or "").strip() + or getpass.getuser() + ) + + return AgentLensConfig( + endpoint=_value("ZHIYANLLM_API_ENDPOINT"), + api_key=_value("ZHIYANLLM_API_KEY"), + app_name=_value("ZHIYANLLM_APP_NAME", DEFAULT_APP_NAME) or DEFAULT_APP_NAME, + business_scenario=_value("ZHIYANLLM_BUSINESS_SCENARIO", DEFAULT_BUSINESS_SCENARIO) + or DEFAULT_BUSINESS_SCENARIO, + user=user or "unknown", + ) + + +def load_runtime() -> dict[str, Any] | None: + """延迟导入可选的 zhiyan 运行时组件。 + + 即使本地没有安装 AgentLens 依赖，hook 主链路也必须继续工作， + 所以这里返回 ``None``，而不是把导入异常抛出去。 + """ + global _RUNTIME + if _RUNTIME is not None: + return _RUNTIME + try: + from zhiyanllm import Zhiyanllm + from zhiyanllm.opentelemetry.instrumentation.semconv_ai import ( + SpanAttributes as ZhiyanSpanAttributes, + ZhiyanllmSpanKindValues, + ) + from zhiyanllm.tracing.context_manager import get_tracer + from zhiyanllm.tracing.manual import track_llm_call, track_task_server_call + from zhiyanllm.tracing.tracing import TracerWrapper + from opentelemetry.context import attach as otel_attach, detach as otel_detach + except Exception: + _RUNTIME = None + return None + + _RUNTIME = { + "Zhiyanllm": Zhiyanllm, + "track_llm_call": track_llm_call, + "track_task_server_call": track_task_server_call, + "TracerWrapper": TracerWrapper, + "get_tracer": get_tracer, + "ZhiyanSpanAttributes": ZhiyanSpanAttributes, + "ZhiyanllmSpanKindValues": ZhiyanllmSpanKindValues, + "otel_attach": otel_attach, + "otel_detach": otel_detach, + } + return _RUNTIME + + +def ensure_initialized(config: AgentLensConfig) -> dict[str, Any] | None: + """仅当有效配置签名发生变化时才重新初始化 zhiyan。""" + global _INIT_SIGNATURE + if not config.enabled: + return None + runtime = load_runtime() + if runtime is None: + return None + signature = (str(config.endpoint), str(config.api_key), str(config.app_name)) + if _INIT_SIGNATURE == signature: + return runtime + runtime["Zhiyanllm"].init( + app_name=str(config.app_name), + api_endpoint=str(config.endpoint), + api_key=str(config.api_key), + disable_batch=True, + ) + _INIT_SIGNATURE = signature + return runtime + + +def project_root_from_state_path(state_path: Path) -> Path: + """尽力从 `logs/.state.json` 反推出项目根目录。""" + try: + current = state_path.resolve().parent + for parent in [current, *current.parents]: + if parent.name == ".codebuddy": + return parent.parent + return state_path.resolve().parents[4] + except Exception: + return Path.cwd().resolve() + + +def debug_enabled(state_path: Path) -> bool: + """判断是否要把 span 调试信息镜像写入 `agentlens-push-debug.ndjson`。""" + root = project_root_from_state_path(state_path) + env_file = read_env_file(root / ".env") + env_local_file = read_env_file(root / ".env.local") + raw = ( + os.environ.get("AGENTLENS_PUSH_DEBUG") + or env_local_file.get("AGENTLENS_PUSH_DEBUG") + or env_file.get("AGENTLENS_PUSH_DEBUG") + or "" + ) + return str(raw).strip().lower() in {"1", "true", "yes", "on"} + + +def debug_write_span( + state_path: Path | None, + sid: str | None, + payload: dict[str, Any], + *, + sanitizer, +) -> None: + """在开启 AgentLens 调试模式时追加写入清洗后的调试记录。""" + if state_path is None or not sid: + return + try: + if not debug_enabled(state_path): + return + debug_path = state_path.parent / "agentlens-push-debug.ndjson" + record = {"sid": sid, **sanitizer(payload)} + with debug_path.open("a", encoding="utf-8") as fp: + fp.write(json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n") + except Exception: + return + + +def set_failure(state_path: Path, sid: str, message: str) -> None: + """记录最近一次 AgentLens 失败，但不打断 hook 主流程。""" + st.update_agentlens_session( + state_path, + sid, + {"enabled": False, "last_error": str(message)}, + ) + + +def get_session_context(state_path: Path, sid: str) -> dict[str, Any]: + """从共享状态里读取 AgentLens sidecar 的 session payload。""" + return st.load_agentlens_session(state_path, sid) diff --git a/.codebuddy/skills/agent-observability/scripts/core/agentlens/normalize.py b/.codebuddy/skills/agent-observability/scripts/core/agentlens/normalize.py new file mode 100644 index 0000000..2b730f0 --- /dev/null +++ b/.codebuddy/skills/agent-observability/scripts/core/agentlens/normalize.py @@ -0,0 +1,323 @@ +from __future__ import annotations +"""AgentLens 负载清洗层。 + +这一层负责“清洗与整形”： +- 在发送或调试前把任意 payload 转成安全的 JSON 结构 +- 把本地 token 结构归一化成类似 OpenAI 的 usage 结构 +- 从 transcript 中重建轻量的 message / tool-call 上下文 +""" + +import json +import os +from pathlib import Path +from typing import Any + +from .. import state as st +from .bootstrap import AgentLensConfig + + +def sanitize_payload(value: Any) -> Any: + """把任意 Python 值转换成适合调试/打点的 JSON 安全结构。""" + try: + return json.loads(json.dumps(value, ensure_ascii=False, default=str)) + except Exception: + return str(value) + + +def association_properties(config: AgentLensConfig, sid: str, state_path: Path | None = None) -> dict[str, str]: + """构造 trace 级关联属性，并允许按 turn 覆盖业务场景。""" + scenario = config.business_scenario + if state_path: + try: + session_ctx = st.load_agentlens_session(state_path, sid) + turn_scenario = session_ctx.get("_turn_business_scenario") + if isinstance(turn_scenario, str) and turn_scenario.strip(): + scenario = turn_scenario.strip() + except Exception: + pass + return { + "session_id": sid, + "business_scenario": scenario, + "user": config.user, + } + + +def to_openai_usage(tokens: dict[str, Any] | None) -> dict[str, int]: + """把本地 token 结构转换成 zhiyan 期望的 usage 形状。""" + if not isinstance(tokens, dict): + return {} + + def _int(v: Any) -> int: + try: + return int(v or 0) + except Exception: + return 0 + + prompt = _int(tokens.get("input")) + completion = _int(tokens.get("output")) + if prompt <= 0 and completion <= 0: + return {} + return {"prompt_tokens": prompt, "completion_tokens": completion, "total_tokens": prompt + completion} + + +def infer_vendor(model: str | None) -> str: + """根据模型名推断一个粗粒度的 provider/vendor 标签。""" + text = str(model or "").strip().lower() + if not text: + return "unknown" + if "gpt" in text or "openai" in text: + return "openai" + if "claude" in text or "anthropic" in text: + return "anthropic" + if "gemini" in text or "google" in text: + return "google" + if text.startswith("hy") or "hunyuan" in text: + return "tencent" + return text.split("/", 1)[0].split("-", 1)[0] or "unknown" + + +def extract_message_text(content: Any) -> str: + """把嵌套的 transcript message 内容压平成纯文本。""" + if content is None: + return "" + if isinstance(content, str): + return content + if isinstance(content, list): + parts: list[str] = [] + for item in content: + if isinstance(item, dict): + txt = item.get("text") + if isinstance(txt, str): + parts.append(txt) + elif item.get("type") == "tool_use": + parts.append(f"[tool_use {item.get('name', '')}]") + elif item.get("type") == "tool_result": + parts.append(extract_message_text(item.get("content"))) + else: + parts.append(str(item)) + return "\n".join(p for p in parts if p) + if isinstance(content, dict): + return extract_message_text(content.get("text") or content.get("content")) + return str(content) + + +def find_message_id_in_record(obj: Any) -> str | None: + """在 transcript 记录里递归查找稳定的 message 标识。""" + message_id_keys = ("messageId", "responseId", "requestId") + if isinstance(obj, dict): + provider = obj.get("providerData") + if isinstance(provider, dict): + for key in message_id_keys: + value = provider.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + extra = obj.get("extra") + if isinstance(extra, dict): + for key in message_id_keys: + value = extra.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + for key in message_id_keys: + value = obj.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + for value in obj.values(): + found = find_message_id_in_record(value) + if found: + return found + elif isinstance(obj, list): + for value in obj: + found = find_message_id_in_record(value) + if found: + return found + return None + + +def collect_tool_calls_for_message(transcript_path: str, message_id: str) -> list[dict[str, Any]]: + """为指定 assistant message 重建它挂载的 tool-call 摘要。""" + if not transcript_path or not message_id or not os.path.isfile(transcript_path): + return [] + try: + with open(transcript_path, "rb") as fp: + blob = fp.read() + except Exception: + return [] + + tool_calls: list[dict[str, Any]] = [] + for raw_line in blob.splitlines(keepends=True): + if not raw_line.lstrip().startswith(b"{"): + continue + try: + rec = json.loads(raw_line.decode("utf-8", errors="ignore")) + except Exception: + continue + if not isinstance(rec, dict): + continue + if str(rec.get("type") or "").strip().lower() != "function_call": + continue + rec_mid = find_message_id_in_record(rec) + if rec_mid != message_id: + continue + call_id = str(rec.get("callId") or "").strip() + name = str(rec.get("name") or "").strip() + arguments = rec.get("arguments") + entry: dict[str, Any] = {"id": call_id, "name": name} + if arguments is not None: + entry["arguments"] = arguments if isinstance(arguments, str) else json.dumps(arguments, ensure_ascii=False) + tool_calls.append(entry) + return tool_calls + + +def find_tool_usage_event(tool_event: dict[str, Any], usage_events: list[dict[str, Any]]) -> dict[str, Any] | None: + """从 usage 列表里挑出最可能属于当前 tool 事件的那一条。""" + tool_name = str(tool_event.get("tool") or "") + transcript_path = str(tool_event.get("transcript_path") or "") + agent = str(tool_event.get("agent") or "main") + for usage_event in usage_events: + if not isinstance(usage_event, dict): + continue + if str(usage_event.get("agent") or "main") != agent: + continue + if transcript_path and str(usage_event.get("transcript_path") or "") != transcript_path: + continue + if tool_name and str(usage_event.get("tool") or "") not in {tool_name, "model_request"}: + continue + if str(usage_event.get("message_id") or "").strip(): + return usage_event + for usage_event in usage_events: + if not isinstance(usage_event, dict): + continue + if str(usage_event.get("agent") or "main") != agent: + continue + if transcript_path and str(usage_event.get("transcript_path") or "") != transcript_path: + continue + return usage_event + return None + + +def build_llm_io_from_transcript( + *, + transcript_path: str, + source_offset: int, + tokens: dict[str, Any] | None, + model: str | None, + start_offset: int = 0, + current_message_id: str | None = None, +) -> tuple[dict[str, Any], dict[str, Any], str]: + """从 transcript 历史中构造紧凑版的 LLM 输入/输出负载。""" + input_data: dict[str, Any] = {"model": str(model or "unknown"), "messages": []} + output_data: dict[str, Any] = {"choices": [], "usage": to_openai_usage(tokens)} + if not transcript_path or not os.path.isfile(transcript_path): + return input_data, output_data, "" + try: + with open(transcript_path, "rb") as fp: + fp.seek(0) + blob = fp.read(source_offset) if source_offset else fp.read() + except Exception: + return input_data, output_data, "" + + messages: list[dict[str, Any]] = [] + system_prompts: list[str] = [] + cursor = 0 + last_asst_text: str | None = None + + def _truncate(text: str, limit: int = 4000) -> str: + if len(text) <= limit: + return text + return text[:limit] + "...(truncated)" + + for raw_line in blob.splitlines(keepends=True): + cursor += len(raw_line) + if not raw_line.lstrip().startswith(b"{"): + continue + try: + rec = json.loads(raw_line.decode("utf-8", errors="ignore")) + except Exception: + continue + if not isinstance(rec, dict): + continue + + rec_type = str(rec.get("type") or "").strip().lower() + is_incremental = cursor > start_offset + + # 重建 prompt 上下文时要跳过“当前正在发出的 assistant message”， + # 否则同一段回复会同时出现在 input 和 output 两边。 + if current_message_id and is_incremental: + rec_mid = find_message_id_in_record(rec) + if rec_mid == current_message_id: + if rec_type == "message" and rec.get("role") == "assistant": + text = extract_message_text(rec.get("content")) + if text: + last_asst_text = text + continue + + if rec_type == "message": + role = rec.get("role") + text = extract_message_text(rec.get("content")) + if not text: + continue + if role == "system": + system_prompts.append(text) + elif is_incremental: + if role == "user": + messages.append({"role": "user", "content": _truncate(text, 8000)}) + last_asst_text = None + elif role == "assistant": + messages.append({"role": "assistant", "content": _truncate(text, 4000)}) + last_asst_text = text + elif is_incremental: + # 把 function call / result 记录转换成 Chat Completions 风格的 + # assistant/tool message，方便 tracing UI 按对话链路展示。 + if rec_type == "function_call": + name = str(rec.get("name") or "").strip() + call_id = str(rec.get("callId") or "").strip() + arguments = rec.get("arguments", "") + if isinstance(arguments, dict): + arguments = json.dumps(arguments, ensure_ascii=False) + tool_call: dict[str, Any] = {"type": "function", "function": {"name": name}} + if name and arguments: + tool_call["function"]["arguments"] = _truncate(str(arguments), 2000) + if call_id: + tool_call["id"] = call_id + if ( + messages + and messages[-1].get("role") == "assistant" + and "tool_calls" not in messages[-1] + and messages[-1].get("content") + ): + messages[-1]["tool_calls"] = [tool_call] + else: + msg: dict[str, Any] = {"role": "assistant", "content": ""} + msg["tool_calls"] = [tool_call] + messages.append(msg) + elif rec_type == "function_call_result": + call_id = str(rec.get("callId") or "").strip() + provider = rec.get("providerData", {}) + result_content = "" + if isinstance(provider, dict): + tool_result = provider.get("toolResult", {}) + if isinstance(tool_result, dict): + result_content = extract_message_text(tool_result.get("content")) + if not result_content: + output = rec.get("output") + result_content = str(output.get("text", "")) if isinstance(output, dict) else "" + tool_msg: dict[str, Any] = {"role": "tool", "content": _truncate(result_content, 2000)} + if call_id: + tool_msg["tool_call_id"] = call_id + messages.append(tool_msg) + + final_messages: list[dict[str, Any]] = [] + for prompt in system_prompts[-2:]: + if prompt: + final_messages.append({"role": "system", "content": _truncate(prompt, 2000)}) + final_messages.extend(messages) + input_data["messages"] = final_messages + + if last_asst_text is not None: + output_data["choices"] = [{ + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": last_asst_text[:8000]}, + }] + + return input_data, output_data, "" diff --git a/.codebuddy/skills/agent-observability/scripts/core/agentlens/runtime.py b/.codebuddy/skills/agent-observability/scripts/core/agentlens/runtime.py new file mode 100644 index 0000000..321f45a --- /dev/null +++ b/.codebuddy/skills/agent-observability/scripts/core/agentlens/runtime.py @@ -0,0 +1,761 @@ +"""AgentLens 运行时编排层。 + +这一层保留对外暴露的 4 个入口： +- `emit_session_start` +- `emit_turn_start` +- `emit_post_step` +- `emit_session_stop` + +它本身不负责底层清洗或 carrier 生成，而是把调用编排到 +`bootstrap / normalize / tracing` 三层上。 +""" +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from opentelemetry import trace as _otel_trace + +from .. import state as st +from .bootstrap import ( + debug_write_span as _debug_write_span, + ensure_initialized as _ensure_initialized, + get_session_context as _get_session_context, + load_config, + project_root as _project_root, + set_failure as _set_failure, +) +from .normalize import ( + association_properties as _association_properties, + build_llm_io_from_transcript as _build_llm_io_from_transcript, + find_tool_usage_event as _find_tool_usage_event, + infer_vendor as _infer_vendor, + sanitize_payload as _sanitize_payload, + to_openai_usage as _to_openai_usage, +) +from .tracing import ( + annotate as _annotate, + emit_assistant_span as _emit_assistant_span, + emit_tool_span as _emit_tool_span, + generate_subagent_carrier as _generate_subagent_carrier, + generate_turn_carrier as _generate_turn_carrier, + resolve_agent_carrier as _resolve_agent_carrier, + resolve_step_carrier as _resolve_step_carrier, + set_agent_aggregate_on_span as _set_agent_aggregate_on_span, + span_context_ids as _span_context_ids, + traceparent_parts as _traceparent_parts, +) + + +def emit_session_start( + *, + state_path: Path, + sid: str, + cwd: str, + agent: str = "main", +) -> None: + """初始化 v2 版本的 `_agentlens` session payload。 + + 在 turn-centric 模型下，这里**不再**生成 trace_id。 + 它只负责记录 session 级元数据（如 `enabled`、`session_id`、`app_name`）， + 并清理上一次运行遗留的 carrier / error 状态。 + 真正的新 trace 会在第一次 `UserPromptSubmit` 时由 `emit_turn_start` 打开。 + """ + _ = agent + config = load_config(_project_root(cwd)) + # 注意：这里不要清空 current_turn / turn_history。 + # 同一个 sid 下 SessionStart 可能多次触发（例如 subagent 启动、IDE 刷新、 + # workspace 切换）。如果这里清空，会把进行中的 turn 擦掉，后续 PostToolUse + # 会全部看到 no_active_turn，最终把这一轮观测链路打断。 + base_updates: dict[str, Any] = { + "session_id": sid, + "app_name": config.app_name, + } + if not config.enabled: + base_updates["enabled"] = False + st.update_agentlens_session( + state_path, + sid, + base_updates, + clear_keys=["carrier", "last_error"], + ) + return + + try: + runtime = _ensure_initialized(config) + if runtime is None: + _set_failure(state_path, sid, "zhiyanllm runtime unavailable") + return + base_updates["enabled"] = True + st.update_agentlens_session( + state_path, + sid, + base_updates, + clear_keys=["carrier", "last_error"], + ) + except Exception as exc: + _set_failure(state_path, sid, str(exc)) + + +def emit_turn_start( + *, + state_path: Path, + sid: str, + cwd: str, + turn_id: str, + prompt_meta: dict[str, Any] | None = None, + business_scenario: str | None = None, +) -> None: + """打开一个新的 turn，生成新的 trace_id 并写入 `_agentlens.current_turn`。 + + 这是当前实现里**唯一**允许生成新 trace_id 的地方。 + `emit_post_step` / `emit_session_stop` 都不会重新造 carrier； + 找不到时只会优雅降级，不会私自开新链路。 + """ + _ = prompt_meta + config = load_config(_project_root(cwd)) + if not config.enabled: + return + + try: + runtime = _ensure_initialized(config) + if runtime is None: + _set_failure(state_path, sid, "zhiyanllm runtime unavailable") + return + # 先把 per-turn business scenario 写进 state， + # 这样 `_generate_turn_carrier -> _association_properties` + # 才能在生成 trace 时把这次 turn 的业务场景烘焙进去。 + if business_scenario: + st.update_agentlens_session( + state_path, + sid, + {"enabled": True, "session_id": sid, "app_name": config.app_name, "_turn_business_scenario": business_scenario}, + clear_keys=["last_error"], + ) + else: + st.update_agentlens_session( + state_path, + sid, + {"enabled": True, "session_id": sid, "app_name": config.app_name}, + clear_keys=["last_error", "_turn_business_scenario"], + ) + carrier = _generate_turn_carrier(runtime, config=config, sid=sid, turn_id=turn_id, state_path=state_path) + st.begin_turn(state_path, sid, turn_id, carrier) + except Exception as exc: + _set_failure(state_path, sid, str(exc)) + + +def emit_post_step( + *, + state_path: Path, + sid: str, + tool_event: dict[str, Any] | None, + usage_events: list[dict[str, Any]], +) -> None: + """发出扁平化的 LLM span，只保留最小 TASK 包装层来传播 carrier 上下文。 + + 面向 turn 的 v2 约束： + - 活跃的 ``current_turn.carrier`` 是唯一的 trace 上下文来源。 + - 如果不存在 ``current_turn``（例如 UserPromptSubmit hook 还没触发）， + 这里只会记录 ``last_error`` 并返回，不会私自生成新的 trace_id。 + - subagent（``active_agent != "main"``）会拿到当前 turn 下的 + ``invoke_agent`` 子 span，后续 chat span 都挂到这个子 span 下， + 以保持因果链路连续。 + """ + if not isinstance(tool_event, dict) and not usage_events: + return + + cwd_hint = "" + if isinstance(tool_event, dict): + cwd_hint = str(tool_event.get("cwd") or "") + config = load_config(_project_root(cwd_hint)) + if not config.enabled: + return + + session_ctx = _get_session_context(state_path, sid) + # 快路径：尊重 SessionStart 阶段已经写入的 enabled 标记。 + if session_ctx.get("enabled") is False: + return + + current_turn = session_ctx.get("current_turn") + if not isinstance(current_turn, dict): + _set_failure(state_path, sid, "no_active_turn") + return + parent_carrier = current_turn.get("carrier") + if not isinstance(parent_carrier, dict) or not parent_carrier.get("traceparent"): + _set_failure(state_path, sid, "no_active_turn_carrier") + return + + try: + runtime = _ensure_initialized(config) + if runtime is None: + _set_failure(state_path, sid, "zhiyanllm runtime unavailable") + return + + tool_name = str((tool_event or {}).get("tool") or "unknown_tool") + active_agent = str((tool_event or {}).get("agent") or "main") + transcript_path = str((tool_event or {}).get("transcript_path") or "") + skills = (tool_event or {}).get("skill") or [] + rules = (tool_event or {}).get("rule") or [] + + # 解析这次事件真正要用的 carrier。 + # main agent 直接使用 turn 根 carrier；subagent 则使用按角色拆分的 + # 子 span carrier，并在当前 turn 下做幂等注册。 + carrier = dict(parent_carrier) + if active_agent and active_agent != "main": + existing = None + subs = current_turn.get("subagent_spans") + if isinstance(subs, dict): + rec = subs.get(active_agent) + if isinstance(rec, dict) and isinstance(rec.get("carrier"), dict): + existing = rec["carrier"] + if isinstance(existing, dict) and existing.get("traceparent"): + carrier = dict(existing) + else: + # 原子化的 get-or-create：只有确认 subagent span 不存在时， + # 才会在写锁内部调用 carrier_factory。 + # 这样可以避免并发 hook 进程之间的 TOCTOU 竞争，产生孤儿 + # invoke_agent.TASK span。 + def _factory() -> dict[str, str]: + return _generate_subagent_carrier( + runtime, parent_carrier=parent_carrier, role=active_agent + ) + + registered = st.upsert_subagent_span_atomic( + state_path, sid, active_agent, carrier_factory=_factory + ) + if isinstance(registered, dict) and registered.get("traceparent"): + carrier = dict(registered) + runtime["Zhiyanllm"].set_association_properties(_association_properties(config, sid, state_path)) + # 静默挂载 carrier 上下文（不额外创建 TASK span），让内部 span + # 继承正确的 trace_id / parent_id。 + _ctx = runtime["TracerWrapper"].extract_context(carrier) + _token = runtime["otel_attach"](_ctx) + try: + preferred_tool_message_id = str((tool_event or {}).get("message_id") or "").strip() or None + _assistant_emitted_mids: set[str] = set() + pre_bumped_steps: set[tuple[str, str]] = set() + committed_steps: set[tuple[str, str]] = set() + + if isinstance(tool_event, dict): + if preferred_tool_message_id and st.get_step_span_carrier( + state_path, sid, active_agent, preferred_tool_message_id + ) is None: + pre_bumped_steps.add((active_agent, preferred_tool_message_id)) + for usage_event in usage_events: + event_agent = str(usage_event.get("agent") or active_agent or "main") + event_message_id = str(usage_event.get("message_id") or "").strip() or None + if event_message_id: + step_key = (event_agent, event_message_id) + if ( + step_key not in pre_bumped_steps + and st.get_step_span_carrier(state_path, sid, event_agent, event_message_id) is None + ): + pre_bumped_steps.add(step_key) + if not usage_events and isinstance(tool_event, dict): + tool_parent_carrier = _resolve_agent_carrier( + state_path=state_path, + sid=sid, + runtime=runtime, + parent_carrier=carrier, + agent=active_agent, + ) + if preferred_tool_message_id: + step_carrier = _resolve_step_carrier( + state_path=state_path, + sid=sid, + runtime=runtime, + parent_carrier=tool_parent_carrier, + agent=active_agent, + message_id=preferred_tool_message_id, + transcript_path=transcript_path, + ) + if isinstance(step_carrier, dict) and step_carrier.get("traceparent"): + tool_parent_carrier = dict(step_carrier) + _emit_tool_span( + runtime, + carrier=tool_parent_carrier, + tool_name=tool_name, + active_agent=active_agent, + skills=skills, + rules=rules, + duration_ms=tool_event.get("ms") if isinstance(tool_event.get("ms"), (int, float)) else None, + tool_details=tool_event.get("tool_details") if isinstance(tool_event.get("tool_details"), dict) else None, + message_id=preferred_tool_message_id, + step_grouping="message_id" if preferred_tool_message_id else "fallback", + state_path=state_path, + sid=sid, + ) + st.bump_agent_aggregate( + state_path, + sid, + active_agent, + tool_event=tool_event, + ) + if ( + preferred_tool_message_id + and (active_agent, preferred_tool_message_id) in pre_bumped_steps + and (active_agent, preferred_tool_message_id) not in committed_steps + ): + st.bump_agent_aggregate( + state_path, + sid, + active_agent, + step_created=True, + ) + committed_steps.add((active_agent, preferred_tool_message_id)) + st.update_agentlens_session(state_path, sid, {"enabled": True}, clear_keys=["last_error"]) + return + + matched_usage = _find_tool_usage_event(tool_event, usage_events) if isinstance(tool_event, dict) else None + tool_message_id = preferred_tool_message_id or str((matched_usage or {}).get("message_id") or "").strip() or None + if isinstance(tool_event, dict): + # 先构造 assistant 预览，再打开 agent/step 父 span。 + # 这样即便 transcript 重建失败，也不会留下空的分组 TASK。 + _assistant_preview_content: str | None = None + _assistant_preview_transcript = transcript_path + _assistant_preview_model = "" + if tool_message_id: + for _ue in usage_events: + if str(_ue.get("message_id") or "").strip() != tool_message_id: + continue + _ue_transcript = str(_ue.get("transcript_path") or transcript_path) + _ue_offset = int(_ue.get("source_offset") or 0) + _, _llm_out_pre, _ = _build_llm_io_from_transcript( + transcript_path=_ue_transcript, + source_offset=_ue_offset, + tokens=_ue.get("tokens"), + model=str(_ue.get("model") or ""), + ) + _pre_content = "" + if _llm_out_pre.get("choices"): + _pre_msg = _llm_out_pre["choices"][0].get("message") or {} + _pre_content = str(_pre_msg.get("content") or "") + if _pre_content and _pre_content.strip(): + _assistant_preview_content = _pre_content + _assistant_preview_transcript = _ue_transcript + _assistant_preview_model = str(_ue.get("model") or "") + break + + tool_parent_carrier = _resolve_agent_carrier( + state_path=state_path, + sid=sid, + runtime=runtime, + parent_carrier=carrier, + agent=active_agent, + ) + if tool_message_id: + step_carrier = _resolve_step_carrier( + state_path=state_path, + sid=sid, + runtime=runtime, + parent_carrier=tool_parent_carrier, + agent=active_agent, + message_id=tool_message_id, + transcript_path=transcript_path, + ) + if isinstance(step_carrier, dict) and step_carrier.get("traceparent"): + tool_parent_carrier = dict(step_carrier) + if _assistant_preview_content and tool_message_id: + _emit_assistant_span( + runtime, + carrier=tool_parent_carrier, + agent=active_agent, + message_id=tool_message_id, + assistant_text=_assistant_preview_content, + model=_assistant_preview_model, + transcript_path=_assistant_preview_transcript, + state_path=state_path, + sid=sid, + ) + _assistant_emitted_mids.add(tool_message_id) + + _emit_tool_span( + runtime, + carrier=tool_parent_carrier, + tool_name=tool_name, + active_agent=active_agent, + skills=skills, + rules=rules, + duration_ms=tool_event.get("ms") if isinstance(tool_event.get("ms"), (int, float)) else None, + tool_details=tool_event.get("tool_details") if isinstance(tool_event.get("tool_details"), dict) else None, + message_id=tool_message_id, + step_grouping="message_id" if tool_message_id else "fallback", + state_path=state_path, + sid=sid, + ) + st.bump_agent_aggregate( + state_path, + sid, + active_agent, + tool_event=tool_event, + ) + if tool_message_id and (active_agent, tool_message_id) in pre_bumped_steps: + step_key = (active_agent, tool_message_id) + if step_key not in committed_steps: + st.bump_agent_aggregate( + state_path, + sid, + active_agent, + step_created=True, + ) + committed_steps.add(step_key) + + _prev_llm_offsets: dict[str, int] = {} + for usage_event in usage_events: + model = str(usage_event.get("model") or "") + ev_transcript = str(usage_event.get("transcript_path") or transcript_path) + ev_offset = int(usage_event.get("source_offset") or 0) + raw_tokens = usage_event.get("tokens") or {} + event_message_id = str(usage_event.get("message_id") or "").strip() or None + event_agent = str(usage_event.get("agent") or active_agent or "main") + start_offset = _prev_llm_offsets.get(ev_transcript) or st.get_last_llm_offset( + st.load_state(state_path), sid, ev_transcript + ) + llm_input, llm_output, _turn_title = _build_llm_io_from_transcript( + transcript_path=ev_transcript, + source_offset=ev_offset, + start_offset=int(start_offset or 0), + current_message_id=event_message_id, + tokens=raw_tokens, + model=model or None, + ) + _prev_llm_offsets[ev_transcript] = ev_offset + # 原子写回 state，供跨进程增量跟踪复用 + def _update_llm_offset(state: dict[str, Any], _ev=ev_transcript, _off=ev_offset) -> None: + st.set_last_llm_offset(state, sid, _off, _ev) + st.update_state_locked(state_path, _update_llm_offset) + # --- 把 skill/rule/agent 作为独立消息标签注入 input_data --- + _meta_messages: list[dict[str, str]] = [] + _ev_agent = str(usage_event.get("agent") or active_agent) + if _ev_agent and _ev_agent != "main": + _meta_messages.append({"role": "agent", "content": _ev_agent}) + if skills: + _meta_messages.append({"role": "skill", "content": ", ".join(skills)}) + if rules: + _meta_messages.append({"role": "rule", "content": ", ".join(rules)}) + if _meta_messages: + llm_input.setdefault("messages", []) + for msg in reversed(_meta_messages): + llm_input["messages"].insert(0, msg) + # --- 构造 usage --- + llm_output["usage"] = _to_openai_usage(raw_tokens) + # --- 只附带成本细节来构造 output 内容 --- + _cost_info = { + "input_tokens": raw_tokens.get("input"), + "output_tokens": raw_tokens.get("output"), + "cache_read": raw_tokens.get("cache_read"), + "cache_creation": raw_tokens.get("cache_creation"), + "total_tokens": raw_tokens.get("total"), + "cost_usd": usage_event.get("cost_usd"), + } + _cost_info = {k: v for k, v in _cost_info.items() if v is not None} + _existing_content = "" + if llm_output.get("choices"): + _msg = llm_output["choices"][0].get("message") or {} + _existing_content = str(_msg.get("content") or "") + _output_content = _existing_content + if _cost_info: + _cost_line = " | ".join(f"{k}: {v}" for k, v in _cost_info.items()) + _output_content = f"[{_cost_line}]\n{_existing_content}" if _existing_content else f"[{_cost_line}]" + llm_output["choices"] = [{ + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": _output_content}, + }] + event_parent_carrier = carrier + if event_message_id: + event_parent_carrier = _resolve_agent_carrier( + state_path=state_path, + sid=sid, + runtime=runtime, + parent_carrier=carrier, + agent=event_agent, + ) + step_carrier = _resolve_step_carrier( + state_path=state_path, + sid=sid, + runtime=runtime, + parent_carrier=event_parent_carrier, + agent=event_agent, + message_id=event_message_id, + transcript_path=ev_transcript, + ) + if isinstance(step_carrier, dict) and step_carrier.get("traceparent"): + event_parent_carrier = dict(step_carrier) + ev_ctx = runtime["TracerWrapper"].extract_context(event_parent_carrier) + ev_token = runtime["otel_attach"](ev_ctx) + # 先累计聚合值，再发 LLM span，这样 span 上带的是最新累计属性。 + st.bump_agent_aggregate( + state_path, + sid, + event_agent, + usage_event=usage_event, + ) + _current_agg = st.get_subagent_aggregate(state_path, sid, event_agent) + try: + with runtime["track_llm_call"](_infer_vendor(model), "model_request") as llm_span: + # track_llm_call 已经设置了 gen_ai.span.kind/system/operation.name。 + # 下方 _annotate 会通过 handle_llm_response 设置 gen_ai.usage.*。 + # LLMSpan 包装层没有 set_attribute，所以这里改用原生 span 写 ID。 + _native_span = _otel_trace.get_current_span() + _set_agent_aggregate_on_span(_native_span, _current_agg) + span_ids = _span_context_ids(_native_span) + parent_parts = _traceparent_parts(event_parent_carrier) + _debug_write_span( + state_path, + sid, + { + "event": "span_emit", + "span_kind": "llm", + "span_name": "model_request", + "model": model, + "message_id": str(event_message_id or ""), + "step_grouping": "message_id" if event_message_id else "fallback", + "agent": event_agent, + "trace_id": span_ids.get("trace_id", parent_parts.get("trace_id", "")), + "span_id": span_ids.get("span_id", ""), + "parent_span_id": parent_parts.get("span_id", ""), + "parent_traceparent": parent_parts.get("traceparent", ""), + "transcript_path": ev_transcript, + "source_offset": ev_offset, + }, + sanitizer=_sanitize_payload, + ) + _annotate( + runtime, + span=llm_span, + input_data=llm_input, + output_data=llm_output, + tags={ + "cost_usd": str(usage_event.get("cost_usd") or "-"), + "input_tokens": str(raw_tokens.get("input", "-")), + "output_tokens": str(raw_tokens.get("output", "-")), + "cache_read": str(raw_tokens.get("cache_read", "-")), + "cache_creation": str(raw_tokens.get("cache_creation", "-")), + "total_tokens": str(raw_tokens.get("total", "-")), + "message_id": str(event_message_id or ""), + "step_grouping": "message_id" if event_message_id else "fallback", + }, + ) + finally: + runtime["otel_detach"](ev_token) + if event_message_id and (event_agent, event_message_id) in pre_bumped_steps: + step_key = (event_agent, event_message_id) + if step_key not in committed_steps: + st.bump_agent_aggregate( + state_path, + sid, + event_agent, + step_created=True, + ) + committed_steps.add(step_key) + if _existing_content and _existing_content.strip() and event_message_id not in _assistant_emitted_mids: + _emit_assistant_span( + runtime, + carrier=event_parent_carrier, + agent=event_agent, + message_id=event_message_id, + assistant_text=_existing_content, + model=model, + transcript_path=ev_transcript, + state_path=state_path, + sid=sid, + ) + finally: + runtime["otel_detach"](_token) + st.update_agentlens_session(state_path, sid, {"enabled": True}, clear_keys=["last_error"]) + except Exception as exc: + _set_failure(state_path, sid, str(exc)) + + +def emit_session_stop( + *, + state_path: Path, + sid: str, + cwd: str, + active_agent: str, + transcript_path: str, + stop_events: list[dict[str, Any]], +) -> None: + """在 session stop 阶段发出扁平化 LLM span，只保留最小 TASK 包装层传播上下文。""" + config = load_config(_project_root(cwd)) + if not config.enabled: + return + + session_ctx = _get_session_context(state_path, sid) + if session_ctx.get("enabled") is False: + return + + current_turn = session_ctx.get("current_turn") + if not isinstance(current_turn, dict): + _set_failure(state_path, sid, "no_active_turn") + return + carrier = current_turn.get("carrier") + if not isinstance(carrier, dict) or not carrier.get("traceparent"): + _set_failure(state_path, sid, "no_active_turn_carrier") + return + + try: + runtime = _ensure_initialized(config) + if runtime is None: + _set_failure(state_path, sid, "zhiyanllm runtime unavailable") + return + + runtime["Zhiyanllm"].set_association_properties(_association_properties(config, sid, state_path)) + if not stop_events: + return + # 静默挂载 carrier 上下文（不额外创建 TASK span）。 + _ctx = runtime["TracerWrapper"].extract_context(carrier) + _token = runtime["otel_attach"](_ctx) + try: + _prev_stop_offsets: dict[str, int] = {} + for stop_event in stop_events: + model = str(stop_event.get("model") or "") + ev_transcript = str(stop_event.get("transcript_path") or transcript_path) + ev_offset = int(stop_event.get("source_offset") or 0) + raw_tokens = stop_event.get("tokens") or {} + event_message_id = str(stop_event.get("message_id") or "").strip() or None + event_agent = str(stop_event.get("agent") or active_agent or "main") + start_offset = _prev_stop_offsets.get(ev_transcript) or st.get_last_llm_offset( + st.load_state(state_path), sid, ev_transcript + ) + llm_input, llm_output, _turn_title = _build_llm_io_from_transcript( + transcript_path=ev_transcript, + source_offset=ev_offset, + start_offset=int(start_offset or 0), + current_message_id=event_message_id, + tokens=raw_tokens, + model=model or None, + ) + _prev_stop_offsets[ev_transcript] = ev_offset + # 原子写回 state，供跨进程增量跟踪复用 + def _update_llm_offset_stop(state: dict[str, Any], _ev=ev_transcript, _off=ev_offset) -> None: + st.set_last_llm_offset(state, sid, _off, _ev) + st.update_state_locked(state_path, _update_llm_offset_stop) + # --- 把 agent 作为独立消息标签注入 input_data --- + _ev_agent = str(stop_event.get("agent") or active_agent) + if _ev_agent and _ev_agent != "main": + llm_input.setdefault("messages", []).insert(0, {"role": "agent", "content": _ev_agent}) + # --- 构造 usage --- + llm_output["usage"] = _to_openai_usage(raw_tokens) + # --- 构造带上下文和成本细节的 output 内容 --- + _cost_info = { + "input_tokens": raw_tokens.get("input"), + "output_tokens": raw_tokens.get("output"), + "cache_read": raw_tokens.get("cache_read"), + "cache_creation": raw_tokens.get("cache_creation"), + "total_tokens": raw_tokens.get("total"), + "cost_usd": stop_event.get("cost_usd"), + "cost_session_usd": stop_event.get("cost_session_usd"), + } + _cost_info = {k: v for k, v in _cost_info.items() if v is not None} + _existing_content = "" + if llm_output.get("choices"): + _msg = llm_output["choices"][0].get("message") or {} + _existing_content = str(_msg.get("content") or "") + _output_content = _existing_content + if _cost_info: + _cost_line = " | ".join(f"{k}: {v}" for k, v in _cost_info.items()) + _output_content = f"[{_cost_line}]\n{_existing_content}" if _existing_content else f"[{_cost_line}]" + llm_output["choices"] = [{ + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": _output_content}, + }] + event_parent_carrier = carrier + # 先解析 agent carrier，让 step 挂在 agent 的 react_agent span 下， + # 而不是直接挂在 turn 根 span 下。 + if event_agent and event_agent != "main": + agent_carrier = _resolve_agent_carrier( + state_path=state_path, + sid=sid, + runtime=runtime, + parent_carrier=carrier, + agent=event_agent, + ) + if isinstance(agent_carrier, dict) and agent_carrier.get("traceparent"): + event_parent_carrier = dict(agent_carrier) + if event_message_id: + step_carrier = _resolve_step_carrier( + state_path=state_path, + sid=sid, + runtime=runtime, + parent_carrier=event_parent_carrier, + agent=event_agent, + message_id=event_message_id, + transcript_path=ev_transcript, + ) + if isinstance(step_carrier, dict) and step_carrier.get("traceparent"): + event_parent_carrier = dict(step_carrier) + ev_ctx = runtime["TracerWrapper"].extract_context(event_parent_carrier) + ev_token = runtime["otel_attach"](ev_ctx) + # 这里不要再 bump，emit_post_step 已经累计过这条 usage_event。 + # SessionStop 会为同一次调用补发一个重复的 LLM span，再累加就会双算。 + # 因此这里只读取当前聚合值。 + _current_agg = st.get_subagent_aggregate(state_path, sid, event_agent) + try: + with runtime["track_llm_call"](_infer_vendor(model), "model_request") as llm_span: + # track_llm_call 已经设置了 gen_ai.span.kind/system/operation.name。 + # 下方 _annotate 会通过 handle_llm_response 设置 gen_ai.usage.*。 + # LLMSpan 包装层没有 set_attribute，所以这里改用原生 span 写 ID。 + _native_span = _otel_trace.get_current_span() + _set_agent_aggregate_on_span(_native_span, _current_agg) + span_ids = _span_context_ids(_native_span) + parent_parts = _traceparent_parts(event_parent_carrier) + _debug_write_span( + state_path, + sid, + { + "event": "span_emit", + "span_kind": "llm", + "span_name": "model_request", + "model": model, + "message_id": str(event_message_id or ""), + "step_grouping": "message_id" if event_message_id else "fallback", + "agent": event_agent, + "trace_id": span_ids.get("trace_id", parent_parts.get("trace_id", "")), + "span_id": span_ids.get("span_id", ""), + "parent_span_id": parent_parts.get("span_id", ""), + "parent_traceparent": parent_parts.get("traceparent", ""), + "transcript_path": ev_transcript, + "source_offset": ev_offset, + }, + sanitizer=_sanitize_payload, + ) + _annotate( + runtime, + span=llm_span, + input_data=llm_input, + output_data=llm_output, + tags={ + "cost_usd": str(stop_event.get("cost_usd") or "-"), + "cost_session_usd": str(stop_event.get("cost_session_usd") or "-"), + "input_tokens": str(raw_tokens.get("input", "-")), + "output_tokens": str(raw_tokens.get("output", "-")), + "cache_read": str(raw_tokens.get("cache_read", "-")), + "cache_creation": str(raw_tokens.get("cache_creation", "-")), + "total_tokens": str(raw_tokens.get("total", "-")), + "message_id": str(event_message_id or ""), + "step_grouping": "message_id" if event_message_id else "fallback", + }, + ) + finally: + runtime["otel_detach"](ev_token) + if _existing_content and _existing_content.strip(): + _emit_assistant_span( + runtime, + carrier=event_parent_carrier, + agent=event_agent, + message_id=event_message_id, + assistant_text=_existing_content, + model=model, + transcript_path=ev_transcript, + state_path=state_path, + sid=sid, + ) + finally: + runtime["otel_detach"](_token) + st.update_agentlens_session(state_path, sid, {"enabled": True}, clear_keys=["last_error"]) + except Exception as exc: + _set_failure(state_path, sid, str(exc)) diff --git a/.codebuddy/skills/agent-observability/scripts/core/agentlens/tracing.py b/.codebuddy/skills/agent-observability/scripts/core/agentlens/tracing.py new file mode 100644 index 0000000..afb678a --- /dev/null +++ b/.codebuddy/skills/agent-observability/scripts/core/agentlens/tracing.py @@ -0,0 +1,504 @@ +from __future__ import annotations +"""AgentLens trace 拓扑层。 + +这一层负责“trace 结构本身”： +- 创建或派生 carrier +- 生成 agent / step / assistant / tool span +- 把后续聚合需要的 span 元数据写回 state +""" + +import json +import secrets +from pathlib import Path +from typing import Any + +from .. import state as st +from .bootstrap import AgentLensConfig, debug_write_span +from .normalize import association_properties, collect_tool_calls_for_message, sanitize_payload + + +def annotate(runtime: dict[str, Any], *, span: Any | None = None, input_data: Any = None, output_data: Any = None, tags: dict[str, Any] | None = None) -> None: + """在调用 zhiyan annotate 前，先统一清洗 payload。""" + _ = span + runtime["Zhiyanllm"].annotate( + input_data=sanitize_payload(input_data), + output_data=sanitize_payload(output_data), + tags=sanitize_payload(tags) if tags else None, + ) + + +def traceparent_parts(carrier: dict[str, str] | None) -> dict[str, str]: + """把 W3C `traceparent` 拆成调试日志里常用的几个字段。""" + traceparent = str((carrier or {}).get("traceparent") or "").strip() + parts = traceparent.split("-") + if len(parts) >= 4: + return { + "traceparent": traceparent, + "trace_id": parts[1], + "span_id": parts[2], + "flags": parts[3], + } + return {"traceparent": traceparent} + + +def span_context_ids(span: Any) -> dict[str, str]: + """从 OTel span 对象中提取十六进制的 trace/span id。""" + try: + ctx = span.get_span_context() + return { + "trace_id": f"{int(ctx.trace_id):032x}", + "span_id": f"{int(ctx.span_id):016x}", + } + except Exception: + return {} + + +def ensure_traceparent(carrier: dict[str, str]) -> None: + """当上游没有成功注入时，补一个最小可用的 `traceparent`。""" + if str(carrier.get("traceparent") or "").strip(): + return + trace_id = secrets.token_hex(16) + parent_id = secrets.token_hex(8) + carrier["traceparent"] = f"00-{trace_id}-{parent_id}-01" + + +def carrier_from_span(parent_carrier: dict[str, str], span: Any) -> dict[str, str]: + """基于父 carrier 和当前 span 生成一个仍在同一 trace 上的子 carrier。""" + try: + span_ctx = span.get_span_context() + trace_id = f"{int(span_ctx.trace_id):032x}" + span_id = f"{int(span_ctx.span_id):016x}" + except Exception: + return {} + parent_tp = str((parent_carrier or {}).get("traceparent") or "").strip() + flags = "01" + if parent_tp: + parts = parent_tp.split("-") + if len(parts) >= 4 and parts[3]: + flags = parts[3] + child_carrier: dict[str, str] = dict(parent_carrier or {}) + child_carrier["traceparent"] = f"00-{trace_id}-{span_id}-{flags}" + return child_carrier + + +def carrier_from_task_span(parent_carrier: dict[str, str], task_span: Any) -> dict[str, str]: + """兼容包装过的 task span 和原始 OTel span 两种形态。""" + span = getattr(task_span, "_span", None) + if span is None and hasattr(task_span, "get_span_context"): + span = task_span + if span is None: + return {} + return carrier_from_span(parent_carrier, span) + + +def zhiyan_attr(runtime: dict[str, Any], name: str, fallback: str) -> str: + """防御式读取 zhiyan 语义约定常量。""" + return str(getattr(runtime.get("ZhiyanSpanAttributes"), name, fallback)) + + +def generate_turn_carrier( + runtime: dict[str, Any], + *, + config: AgentLensConfig, + sid: str, + turn_id: str, + state_path: Path | None = None, +) -> dict[str, str]: + """为一个用户 turn 生成根 carrier。""" + from opentelemetry import trace + + carrier: dict[str, str] = {} + span = trace.get_tracer(__name__).start_span(f"turn.{sid}.{turn_id}") + try: + runtime["Zhiyanllm"].set_association_properties(association_properties(config, sid, state_path)) + runtime["Zhiyanllm"].inject_context(carrier) + finally: + span.end() + ensure_traceparent(carrier) + return carrier + + +def generate_subagent_carrier(runtime: dict[str, Any], *, parent_carrier: dict[str, str], role: str) -> dict[str, str]: + """在当前 turn 下面创建 `invoke_agent` 子 carrier。""" + try: + with runtime["track_task_server_call"]("invoke_agent", carrier=parent_carrier) as task_span: + annotate(runtime, input_data={"agent": role}, output_data={"result": "dispatched"}, tags={"agent": role}) + child_carrier = carrier_from_task_span(parent_carrier, task_span) + if child_carrier: + return child_carrier + except Exception: + return {} + return {} + + +def generate_step_carrier( + runtime: dict[str, Any], + *, + parent_carrier: dict[str, str], + agent: str, + message_id: str, + transcript_path: str, +) -> dict[str, str]: + """创建用于归并同一条 message 工作的 synthetic step span。""" + try: + with runtime["track_task_server_call"]("react_step", carrier=parent_carrier) as task_span: + annotate( + runtime, + input_data={"agent": agent, "message_id": message_id}, + output_data={"result": "grouped"}, + tags={"message_id": message_id, "agent": agent, "transcript_path": transcript_path or ""}, + ) + child_carrier = carrier_from_task_span(parent_carrier, task_span) + if child_carrier: + return child_carrier + except Exception: + return {} + return {} + + +def generate_agent_carrier(runtime: dict[str, Any], *, parent_carrier: dict[str, str], agent: str) -> dict[str, str]: + """在当前 turn 下创建稳定的 per-agent 聚合 span。""" + try: + with runtime["track_task_server_call"]("react_agent", carrier=parent_carrier) as task_span: + span = getattr(task_span, "_span", None) + if span is not None and hasattr(span, "set_attribute"): + span.set_attribute("agent.name", agent) + annotate(runtime, input_data={"agent": agent}, output_data={"agent": agent}, tags={"agent": agent}) + child_carrier = carrier_from_task_span(parent_carrier, task_span) + if child_carrier: + return child_carrier + except Exception: + return {} + return {} + + +def resolve_agent_carrier( + *, + state_path: Path, + sid: str, + runtime: dict[str, Any], + parent_carrier: dict[str, str], + agent: str, +) -> dict[str, str]: + """从共享状态里获取或创建持久化的 agent 聚合 carrier。""" + normalized_agent = str(agent or "main").strip() or "main" + existing = st.get_agent_span_carrier(state_path, sid, normalized_agent) + if isinstance(existing, dict) and existing.get("traceparent"): + return dict(existing) + + def _factory() -> dict[str, str]: + return generate_agent_carrier(runtime, parent_carrier=parent_carrier, agent=normalized_agent) + + effective_carrier = st.upsert_agent_span_atomic(state_path, sid, normalized_agent, carrier_factory=_factory) + if not isinstance(effective_carrier, dict) or not effective_carrier.get("traceparent"): + return dict(parent_carrier) + agent_parts = traceparent_parts(effective_carrier) + parent_parts = traceparent_parts(parent_carrier) + debug_write_span( + state_path, + sid, + { + "event": "span_emit", + "span_kind": "agent", + "span_name": "react_agent", + "agent": normalized_agent, + "trace_id": agent_parts.get("trace_id", parent_parts.get("trace_id", "")), + "span_id": agent_parts.get("span_id", ""), + "parent_span_id": parent_parts.get("span_id", ""), + "parent_traceparent": parent_parts.get("traceparent", ""), + }, + sanitizer=sanitize_payload, + ) + return dict(effective_carrier) + + +def resolve_step_carrier( + *, + state_path: Path, + sid: str, + runtime: dict[str, Any], + parent_carrier: dict[str, str], + agent: str, + message_id: str | None, + transcript_path: str, +) -> dict[str, str] | None: + """获取或创建按 message 分组使用的 step carrier。""" + normalized_message_id = str(message_id or "").strip() + if not normalized_message_id: + return None + existing = st.get_step_span_carrier(state_path, sid, agent, normalized_message_id) + if isinstance(existing, dict) and existing.get("traceparent"): + return dict(existing) + + def _factory() -> dict[str, str]: + return generate_step_carrier( + runtime, + parent_carrier=parent_carrier, + agent=agent, + message_id=normalized_message_id, + transcript_path=transcript_path, + ) + + effective_carrier = st.upsert_step_span_atomic( + state_path, + sid, + agent, + normalized_message_id, + carrier_factory=_factory, + transcript_path=transcript_path, + ) + if not isinstance(effective_carrier, dict) or not effective_carrier.get("traceparent"): + return None + step_parts = traceparent_parts(effective_carrier) + parent_parts = traceparent_parts(parent_carrier) + debug_write_span( + state_path, + sid, + { + "event": "span_emit", + "span_kind": "step", + "span_name": "react_step", + "message_id": normalized_message_id, + "agent": agent, + "trace_id": step_parts.get("trace_id", parent_parts.get("trace_id", "")), + "span_id": step_parts.get("span_id", ""), + "parent_span_id": parent_parts.get("span_id", ""), + "parent_traceparent": parent_parts.get("traceparent", ""), + "transcript_path": transcript_path or "", + }, + sanitizer=sanitize_payload, + ) + return dict(effective_carrier) + + +def set_agent_aggregate_on_span(span: Any, aggregate: dict[str, Any] | None) -> None: + """把 agent 的累计指标投影成当前 span 的属性。""" + if span is None or not hasattr(span, "set_attribute") or not isinstance(aggregate, dict): + return + tokens = aggregate.get("tokens") or {} + if isinstance(tokens, dict): + for key in ("input", "output", "cache_read", "cache_creation", "total"): + if key in tokens: + span.set_attribute(f"agent.cumulative_tokens.{key}", int(tokens.get(key) or 0)) + for key in ("cost_usd", "tool_duration_ms", "llm_call_count", "tool_call_count", "step_count", "event_count"): + if key in aggregate: + try: + span.set_attribute(f"agent.cumulative.{key}", int(aggregate.get(key) or 0)) + except (TypeError, ValueError): + span.set_attribute(f"agent.cumulative.{key}", str(aggregate.get(key))) + + +def emit_assistant_span( + runtime: dict[str, Any], + *, + carrier: dict[str, str], + agent: str, + message_id: str | None, + assistant_text: str, + model: str | None = None, + transcript_path: str | None = None, + state_path: Path | None = None, + sid: str | None = None, +) -> None: + """发出一个轻量 assistant span，并按需补上 tool calls。""" + if not assistant_text or not assistant_text.strip(): + return + tool_calls: list[dict[str, Any]] = [] + if message_id and transcript_path: + tool_calls = collect_tool_calls_for_message(transcript_path, message_id) + output_data: dict[str, Any] = {"role": "assistant", "content": assistant_text[:8000]} + if tool_calls: + output_data["tool_calls"] = tool_calls + ctx = runtime["TracerWrapper"].extract_context(carrier) + token = runtime["otel_attach"](ctx) + try: + with runtime["get_tracer"]() as tracer: + with tracer.start_as_current_span(name="assistant_message") as span: + span.set_attribute(runtime["ZhiyanSpanAttributes"].LLM_SPAN_KIND, runtime["ZhiyanllmSpanKindValues"].LLM.value) + span_ids = span_context_ids(span) + parent_parts = traceparent_parts(carrier) + debug_write_span( + state_path, + sid, + { + "event": "span_emit", + "span_kind": "assistant", + "span_name": "assistant_message", + "message_id": str(message_id or ""), + "agent": agent, + "model": str(model or ""), + "tool_call_count": len(tool_calls), + "trace_id": span_ids.get("trace_id", parent_parts.get("trace_id", "")), + "span_id": span_ids.get("span_id", ""), + "parent_span_id": parent_parts.get("span_id", ""), + "parent_traceparent": parent_parts.get("traceparent", ""), + }, + sanitizer=sanitize_payload, + ) + annotate( + runtime, + input_data={"agent": agent, "message_id": message_id or "", "model": model or ""}, + output_data=output_data, + tags={"message_id": str(message_id or ""), "agent": agent}, + ) + finally: + runtime["otel_detach"](token) + + +def emit_tool_span( + runtime: dict[str, Any], + *, + carrier: dict[str, str], + tool_name: str, + active_agent: str, + skills: list[str], + rules: list[str], + duration_ms: int | None = None, + tool_details: dict[str, Any] | None = None, + message_id: str | None = None, + step_grouping: str = "fallback", + state_path: Path | None = None, + sid: str | None = None, +) -> None: + """发出 tool span，并以受控大小挂载 tool payload。""" + def _truncate(value: Any, limit: int = 1200) -> Any: + text = value if isinstance(value, str) else None + if text is None: + return value + if len(text) <= limit: + return text + return text[:limit] + "...(truncated)" + + def _maybe_parse_json_text(value: Any) -> Any: + if not isinstance(value, str): + return value + text = value.strip() + if not text or text[0] not in "{[": + return value + try: + return json.loads(text) + except Exception: + return value + + def _parse_bash_result_content(value: Any) -> dict[str, Any] | None: + if not isinstance(value, str) or "Command:" not in value: + return None + normalized_value = value.replace("\\n", "\n") + markers = [ + ("Command:", "command"), + ("Stdout:", "stdout"), + ("Stderr:", "stderr"), + ("Exit Code:", "exit_code"), + ("Signal:", "signal"), + ] + parsed: dict[str, Any] = {} + for idx, (marker, key) in enumerate(markers): + start = normalized_value.find(marker) + if start < 0: + continue + start += len(marker) + end = len(normalized_value) + for next_marker, _ in markers[idx + 1:]: + pos = normalized_value.find(next_marker, start) + if pos >= 0: + end = min(end, pos) + segment = normalized_value[start:end].strip() + if not segment: + continue + parsed[key] = _truncate(segment, 1200 if key in {"command", "stdout", "stderr"} else 200) + return parsed or None + + tool_input: dict[str, Any] = {"tool": tool_name, "agent": active_agent} + if skills: + tool_input["skill"] = ", ".join(skills) + if rules: + tool_input["rule"] = ", ".join(rules) + if message_id: + tool_input["message_id"] = message_id + details = dict(tool_details or {}) + call_id = str(details.get("call_id") or "").strip() or None + if call_id: + tool_input["call_id"] = call_id + arguments_display_text = details.get("arguments_display_text") + if arguments_display_text: + tool_input["arguments_display_text"] = _truncate(arguments_display_text, 600) + arguments = details.get("arguments") + if arguments: + parsed_arguments = _maybe_parse_json_text(arguments) + tool_input["arguments"] = _truncate(parsed_arguments, 1200) + if isinstance(parsed_arguments, dict): + if parsed_arguments.get("command"): + tool_input["command"] = _truncate(parsed_arguments.get("command"), 1200) + if parsed_arguments.get("description"): + tool_input["description"] = _truncate(parsed_arguments.get("description"), 400) + + tool_output: dict[str, Any] = {"result": "executed"} + result_content = details.get("result_content") + if result_content is not None: + structured_result = _parse_bash_result_content(result_content) + if structured_result: + tool_output["result"] = structured_result + else: + tool_output["result_content"] = _truncate(result_content, 1600) + output_text = details.get("output_text") + if output_text: + tool_output["output_text"] = _truncate(output_text, 1600) + raw_response = details.get("raw_response") + if isinstance(raw_response, dict): + tool_output["raw_response"] = { + "exitCode": raw_response.get("exitCode"), + "signal": raw_response.get("signal"), + "interrupted": raw_response.get("interrupted"), + "sandboxDenied": raw_response.get("sandboxDenied"), + "tool_error_code": raw_response.get("tool_error_code"), + } + + ctx = runtime["TracerWrapper"].extract_context(carrier) + token = runtime["otel_attach"](ctx) + try: + with runtime["get_tracer"]() as tracer: + with tracer.start_as_current_span(name=f"{tool_name}.TOOL") as span: + span_kind_value = runtime["ZhiyanllmSpanKindValues"].TOOL.value + span.set_attribute(zhiyan_attr(runtime, "LLM_SPAN_KIND", "gen_ai.span.kind"), span_kind_value) + span.set_attribute("gen_ai.span.kind", span_kind_value) + span.set_attribute(zhiyan_attr(runtime, "TOOL_NAME", "tool.name"), tool_name) + if isinstance(tool_details, dict): + desc = tool_details.get("description") or tool_details.get("tool_description") + if isinstance(desc, str) and desc.strip(): + span.set_attribute(zhiyan_attr(runtime, "TOOL_DESCRIPTION", "tool.description"), desc.strip()) + tool_params = tool_input.get("arguments") + if tool_params is not None: + if not isinstance(tool_params, str): + tool_params = json.dumps(tool_params, ensure_ascii=False, default=str) + span.set_attribute(zhiyan_attr(runtime, "TOOL_PARAMETERS", "tool.parameters"), tool_params) + if duration_ms is not None: + span.set_attribute("tool.duration_ms", int(duration_ms)) + span_ids = span_context_ids(span) + parent_parts = traceparent_parts(carrier) + debug_write_span( + state_path, + sid, + { + "event": "span_emit", + "span_kind": "tool", + "span_name": f"{tool_name}.TOOL", + "tool": tool_name, + "message_id": str(message_id or ""), + "call_id": call_id or "", + "step_grouping": step_grouping, + "agent": active_agent, + "trace_id": span_ids.get("trace_id", parent_parts.get("trace_id", "")), + "span_id": span_ids.get("span_id", ""), + "parent_span_id": parent_parts.get("span_id", ""), + "parent_traceparent": parent_parts.get("traceparent", ""), + }, + sanitizer=sanitize_payload, + ) + annotate( + runtime, + input_data=tool_input, + output_data=tool_output, + tags={"step_grouping": step_grouping, "message_id": str(message_id or "")}, + ) + finally: + runtime["otel_detach"](token) diff --git a/.codebuddy/skills/agent-observability/scripts/core/cls_sink.py b/.codebuddy/skills/agent-observability/scripts/core/cls_sink.py new file mode 100644 index 0000000..6525f6e --- /dev/null +++ b/.codebuddy/skills/agent-observability/scripts/core/cls_sink.py @@ -0,0 +1,470 @@ +from __future__ import annotations + +import hashlib +import hmac +import json +import os +import struct +import subprocess +import time +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from .agentlens import bootstrap + + + +@dataclass(frozen=True) +class CLSConfig: + enabled: bool + endpoint: str + topic_id: str + secret_id: str + secret_key: str + secret_token: str + service_name: str + timeout_seconds: int + helper_path: Path + sdk_entry_path: Path + + @property + def ready(self) -> bool: + return bool( + self.enabled + and self.endpoint + and self.topic_id + and self.secret_id + and self.secret_key + and self.helper_path.is_file() + and self.sdk_entry_path.is_file() + ) + + +def repo_root() -> Path: + return Path(__file__).resolve().parents[5] + + +def helper_path() -> Path: + return Path(__file__).with_name("cls_uploader.mjs") + + +def sdk_entry_path() -> Path: + return ( + repo_root() + / "cls-codebuddy" + / "tencentcloud-cls-sdk-codebuddy" + / "node_modules" + / "tencentcloud-cls-sdk-js" + / "dist" + / "index.js" + ) + + +def debug_log_path() -> Path: + return Path(__file__).resolve().parents[2] / "logs" / "cls-push-debug.ndjson" + + +def _value( + name: str, + *, + env_local: dict[str, str], + env_file: dict[str, str], + default: str = "", + aliases: tuple[str, ...] = (), +) -> str: + keys = (name, *aliases) + for key in keys: + raw = os.environ.get(key) + if raw is not None and str(raw).strip(): + return str(raw).strip() + for key in keys: + raw = env_local.get(key) + if raw is not None and str(raw).strip(): + return str(raw).strip() + for key in keys: + raw = env_file.get(key) + if raw is not None and str(raw).strip(): + return str(raw).strip() + return default + + +def _enabled_flag(*, env_local: dict[str, str], env_file: dict[str, str]) -> bool: + raw = _value( + "CLS_CODINGAGENT_ENABLED", + env_local=env_local, + env_file=env_file, + default="1", + ) + return str(raw).strip().lower() not in {"0", "false", "off", "no"} + + +def _debug_write(payload: dict[str, Any]) -> None: + try: + path = debug_log_path() + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as fp: + fp.write(json.dumps(payload, ensure_ascii=False) + "\n") + except Exception: + return + + +# ---- CLS PutLogs 的 protobuf 编码 ---- +# CLS 使用了简化版 protobuf： +# LogGroup { repeated Log logs = 1; optional string filename = 2; } +# Log { optional uint32 time = 1; repeated Content contents = 2; } +# Content { optional string key = 1; optional string value = 2; } + +def _encode_varint(value: int) -> bytes: + result = b"" + while value > 0x7F: + result += bytes([(value & 0x7F) | 0x80]) + value >>= 7 + result += bytes([value & 0x7F]) + return result + +def _encode_field(field_number: int, wire_type: int, data: bytes) -> bytes: + tag = (field_number << 3) | wire_type + return _encode_varint(tag) + data + +def _encode_string_field(field_number: int, value: str) -> bytes: + encoded = value.encode("utf-8") + return _encode_field(field_number, 2, _encode_varint(len(encoded)) + encoded) + +def _encode_uint32_field(field_number: int, value: int) -> bytes: + return _encode_field(field_number, 0, _encode_varint(value)) + +def _encode_content(key: str, value: str) -> bytes: + msg = b"" + if key: + msg += _encode_string_field(1, key) + if value: + msg += _encode_string_field(2, value) + return msg + +def _encode_log(record: dict[str, Any], service_name: str) -> bytes: + ts = record.get("ts") + if isinstance(ts, (int, float)): + log_time = int(ts) if ts < 1_000_000_000_000 else int(ts / 1000) + else: + log_time = int(time.time()) + + msg = _encode_uint32_field(1, log_time) + + merged = {"service_name": service_name, **record} + for key, value in merged.items(): + if value is None: + continue + if isinstance(value, (dict, list)): + value = json.dumps(value, ensure_ascii=False) + else: + value = str(value) + msg += _encode_field(2, 2, _encode_varint(len(_encode_content(key, value))) + _encode_content(key, value)) + + return msg + +def _encode_log_group(records: list[dict[str, Any]], service_name: str) -> bytes: + msg = b"" + for record in records: + log_bytes = _encode_log(record, service_name) + msg += _encode_field(1, 2, _encode_varint(len(log_bytes)) + log_bytes) + if service_name: + msg += _encode_string_field(2, service_name) + return msg + +def _encode_log_group_list(records: list[dict[str, Any]], service_name: str) -> bytes: + """编码 LogGroupList protobuf：`message LogGroupList { repeated LogGroup logGroupList = 1; }`。""" + log_group_bytes = _encode_log_group(records, service_name) + return _encode_field(1, 2, _encode_varint(len(log_group_bytes)) + log_group_bytes) + + +# ---- CLS API v3 使用的 TC3-HMAC-SHA256 签名 ---- + +def _hmac_sha256(key: bytes, data: str) -> bytes: + return hmac.new(key, data.encode("utf-8"), hashlib.sha256).digest() + +def _sha256_hex(data: str) -> str: + return hashlib.sha256(data.encode("utf-8")).hexdigest() + +def _put_logs_via_api( + endpoint: str, + topic_id: str, + secret_id: str, + secret_key: str, + log_group_bytes: bytes, + region: str = "ap-guangzhou", + timeout: int = 20, +) -> tuple[bool, str]: + """通过 CLS API v3 发送 UploadLog，并使用 TC3-HMAC-SHA256 签名。 + + `log_group_bytes` 应该是一个 LogGroupList protobuf。 + """ + host = endpoint + service = "cls" + action = "UploadLog" + version = "2020-10-16" + algorithm = "TC3-HMAC-SHA256" + content_type = "application/octet-stream" + + timestamp = int(time.time()) + date_str = time.strftime("%Y-%m-%d", time.gmtime(timestamp)) + + # 请求体就是原始 protobuf 字节串（LogGroupList） + payload = log_group_bytes + hashed_payload = hashlib.sha256(payload).hexdigest() + + # 规范化请求串 + canonical_headers = f"content-type:{content_type}\nhost:{host}\nx-tc-action:{action.lower()}\n" + signed_headers = "content-type;host;x-tc-action" + canonical_request = f"POST\n/\n\n{canonical_headers}\n{signed_headers}\n{hashed_payload}" + + # 待签名字符串 + credential_scope = f"{date_str}/{service}/tc3_request" + hashed_canonical_request = hashlib.sha256(canonical_request.encode("utf-8")).hexdigest() + string_to_sign = f"{algorithm}\n{timestamp}\n{credential_scope}\n{hashed_canonical_request}" + + # 签名结果 + secret_date = _hmac_sha256(("TC3" + secret_key).encode("utf-8"), date_str) + secret_service = _hmac_sha256(secret_date, service) + secret_signing = _hmac_sha256(secret_service, "tc3_request") + signature = hmac.new(secret_signing, string_to_sign.encode("utf-8"), hashlib.sha256).hexdigest() + + authorization = f"{algorithm} Credential={secret_id}/{credential_scope}, SignedHeaders={signed_headers}, Signature={signature}" + + # 构造 HTTP 请求 + url = f"https://{host}" + headers = { + "Authorization": authorization, + "Content-Type": content_type, + "Host": host, + "X-TC-Action": action, + "X-TC-Timestamp": str(timestamp), + "X-TC-Version": version, + "X-TC-Region": region, + "X-CLS-TopicId": topic_id, + "Content-Length": str(len(payload)), + } + + req = urllib.request.Request(url, data=payload, headers=headers, method="POST") + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + body = resp.read().decode("utf-8", errors="replace") + if resp.status == 200: + return True, body + else: + return False, f"HTTP {resp.status}: {body}" + except urllib.error.HTTPError as e: + body = e.read().decode("utf-8", errors="replace")[:500] + return False, f"HTTP {e.code}: {body}" + except Exception as e: + return False, f"{type(e).__name__}: {e}" + + +def _int_value( + name: str, + *, + env_local: dict[str, str], + env_file: dict[str, str], + default: int, + aliases: tuple[str, ...] = (), +) -> int: + raw = _value(name, env_local=env_local, env_file=env_file, default=str(default), aliases=aliases) + try: + parsed = int(raw) + except Exception: + return default + return parsed if parsed > 0 else default + + +def load_config(cwd: str | None = None) -> CLSConfig: + project_root = bootstrap.project_root(cwd) + env_file = bootstrap.read_env_file(project_root / ".env") + env_local = bootstrap.read_env_file(project_root / ".env.local") + return CLSConfig( + enabled=_enabled_flag(env_local=env_local, env_file=env_file), + endpoint=_value( + "CLS_ENDPOINT", + env_local=env_local, + env_file=env_file, + ), + topic_id=_value( + "CLS_TOPIC_ID", + env_local=env_local, + env_file=env_file, + ), + secret_id=_value( + "CLS_SECRET_ID", + env_local=env_local, + env_file=env_file, + aliases=( + "TC_SECRET_ID", + "TENCENTCLOUD_SECRET_ID", + "TENCENTCLOUD_SECRET_ID_438167613", + ), + ), + secret_key=_value( + "CLS_SECRET_KEY", + env_local=env_local, + env_file=env_file, + aliases=( + "TC_SECRET_KEY", + "TENCENTCLOUD_SECRET_KEY", + "TENCENTCLOUD_SECRET_KEY_438167613", + ), + ), + secret_token=_value( + "CLS_SECRET_TOKEN", + env_local=env_local, + env_file=env_file, + aliases=("TC_SECRET_TOKEN", "TC_SESSION_TOKEN", "CLS_SESSION_TOKEN"), + ), + service_name=_value( + "CLS_SERVICE_NAME", + env_local=env_local, + env_file=env_file, + ), + timeout_seconds=_int_value( + "CLS_UPLOAD_TIMEOUT_SECONDS", + env_local=env_local, + env_file=env_file, + default=0, + aliases=("CLS_TIMEOUT_SECONDS",), + ), + helper_path=helper_path(), + sdk_entry_path=sdk_entry_path(), + ) + + +def mirror_record(record: dict[str, Any], *, cwd: str | None = None) -> bool: + config = load_config(cwd) + if not config.ready: + _debug_write( + { + "stage": "config_not_ready", + "endpoint": config.endpoint, + "topic_id": config.topic_id, + "secret_id_present": bool(config.secret_id), + "secret_key_present": bool(config.secret_key), + "secret_token_present": bool(config.secret_token), + "timeout_seconds": config.timeout_seconds, + "helper_exists": config.helper_path.is_file(), + "sdk_exists": config.sdk_entry_path.is_file(), + "event": record.get("event"), + "sid": record.get("sid"), + } + ) + return False + + _debug_write( + { + "stage": "uploader_start", + "endpoint": config.endpoint, + "topic_id": config.topic_id, + "timeout_seconds": config.timeout_seconds, + "event": record.get("event"), + "sid": record.get("sid"), + } + ) + + # 先尝试 Python 原生 API v3 上报（兼容内网 endpoint） + try: + log_group_bytes = _encode_log_group_list([record], config.service_name) + success, detail = _put_logs_via_api( + endpoint=config.endpoint, + topic_id=config.topic_id, + secret_id=config.secret_id, + secret_key=config.secret_key, + log_group_bytes=log_group_bytes, + timeout=config.timeout_seconds, + ) + if success: + _debug_write( + { + "stage": "uploader_ok", + "endpoint": config.endpoint, + "topic_id": config.topic_id, + "method": "python_api_v3", + "event": record.get("event"), + "sid": record.get("sid"), + } + ) + return True + else: + _debug_write( + { + "stage": "uploader_failed", + "method": "python_api_v3", + "error": detail[:2000], + "timeout_seconds": config.timeout_seconds, + "event": record.get("event"), + "sid": record.get("sid"), + } + ) + except Exception as err: + _debug_write( + { + "stage": "uploader_exception", + "method": "python_api_v3", + "error": f"{type(err).__name__}: {err}", + "timeout_seconds": config.timeout_seconds, + "event": record.get("event"), + "sid": record.get("sid"), + } + ) + + # 回退方案：使用 node SDK uploader + payload = { + "endpoint": config.endpoint, + "topicId": config.topic_id, + "secretId": config.secret_id, + "secretKey": config.secret_key, + "secretToken": config.secret_token, + "serviceName": config.service_name, + "records": [record], + } + try: + completed = subprocess.run( + ["node", str(config.helper_path), str(config.sdk_entry_path)], + input=json.dumps(payload, ensure_ascii=False), + text=True, + capture_output=True, + check=False, + timeout=config.timeout_seconds, + ) + if completed.returncode != 0: + _debug_write( + { + "stage": "uploader_failed", + "returncode": completed.returncode, + "stdout": (completed.stdout or "")[:2000], + "stderr": (completed.stderr or "")[:2000], + "timeout_seconds": config.timeout_seconds, + "event": record.get("event"), + "sid": record.get("sid"), + } + ) + return False + except Exception as err: + _debug_write( + { + "stage": "uploader_exception", + "error": f"{type(err).__name__}: {err}", + "timeout_seconds": config.timeout_seconds, + "event": record.get("event"), + "sid": record.get("sid"), + } + ) + return False + + _debug_write( + { + "stage": "uploader_ok", + "endpoint": config.endpoint, + "topic_id": config.topic_id, + "timeout_seconds": config.timeout_seconds, + "event": record.get("event"), + "sid": record.get("sid"), + } + ) + return True diff --git a/.codebuddy/skills/agent-observability/scripts/core/cls_uploader.mjs b/.codebuddy/skills/agent-observability/scripts/core/cls_uploader.mjs new file mode 100644 index 0000000..5c5dccc --- /dev/null +++ b/.codebuddy/skills/agent-observability/scripts/core/cls_uploader.mjs @@ -0,0 +1,96 @@ +import { createRequire } from 'node:module'; + +const MAX_FIELD_LENGTH = 32 * 1024; + +function truncate(value) { + if (value.length <= MAX_FIELD_LENGTH) { + return value; + } + return `${value.slice(0, MAX_FIELD_LENGTH)}...[truncated]`; +} + +function stringifyValue(value) { + if (value === null || value === undefined) { + return ''; + } + if (typeof value === 'string') { + return truncate(value); + } + if (typeof value === 'number' || typeof value === 'boolean') { + return String(value); + } + try { + return truncate(JSON.stringify(value)); + } catch { + return truncate(String(value)); + } +} + +function timestampSeconds(record) { + const raw = Number(record?.ts); + if (!Number.isFinite(raw) || raw <= 0) { + return Math.floor(Date.now() / 1000); + } + return raw > 1_000_000_000_000 ? Math.floor(raw / 1000) : Math.floor(raw); +} + +async function readStdin() { + const chunks = []; + for await (const chunk of process.stdin) { + chunks.push(chunk); + } + return Buffer.concat(chunks).toString('utf8'); +} + +async function main() { + const sdkPath = process.argv[2]; + if (!sdkPath) { + process.exitCode = 2; + process.stderr.write('missing sdk path\n'); + return; + } + + const require = createRequire(import.meta.url); + const { AsyncClient, LogItem, Content, LogGroup, PutLogsRequest } = require(sdkPath); + + const raw = await readStdin(); + const payload = JSON.parse(raw || '{}'); + const records = Array.isArray(payload.records) ? payload.records : []; + if (!payload.endpoint || !payload.topicId || !payload.secretId || !payload.secretKey || records.length === 0) { + process.exitCode = 0; + return; + } + + const client = new AsyncClient({ + endpoint: payload.endpoint, + secretId: payload.secretId, + secretKey: payload.secretKey, + secretToken: payload.secretToken || '', + sourceIp: '127.0.0.1', + retry_times: 3, + }); + + const logGroup = new LogGroup(); + logGroup.setFilename(payload.serviceName || 'agent-observability'); + + for (const record of records) { + const item = new LogItem(); + const merged = { + service_name: payload.serviceName || 'agent-observability', + ...record, + }; + for (const [key, value] of Object.entries(merged)) { + item.pushBack(new Content(key, stringifyValue(value))); + } + item.setTime(timestampSeconds(record)); + logGroup.addLogs(item); + } + + const request = new PutLogsRequest(payload.topicId, logGroup); + await client.PutLogs(request); +} + +main().catch((error) => { + process.exitCode = 1; + process.stderr.write(`${error?.message || String(error)}\n`); +}); diff --git a/.codebuddy/skills/agent-observability/scripts/core/collector.py b/.codebuddy/skills/agent-observability/scripts/core/collector.py new file mode 100644 index 0000000..cfd681b --- /dev/null +++ b/.codebuddy/skills/agent-observability/scripts/core/collector.py @@ -0,0 +1,1008 @@ +"""通用采集层。 + +这一层负责最基础的观测数据采集： +- 维护 Pre/PostToolUse 的配对关系，计算工具耗时 +- 解析 transcript 增量，提取 usage / model / tool 记录 +- 把 skill/rule 命中情况写入 session state +""" +from __future__ import annotations + +import json +import os +import time +from pathlib import Path +from typing import Any + +from . import agent_identity, scanner, state as st + +# 需要从 transcript 中提取的 token 字段 +TOKEN_KEYS = ( + "input_tokens", "output_tokens", "cache_read_input_tokens", + "cache_creation_input_tokens", "total_tokens", +) + +# transcript 记录里可能携带模型名的字段 +MODEL_KEYS = ("model", "requestModelName", "requestModelId") +MESSAGE_ID_KEYS = ("messageId", "responseId", "requestId") + + +def transcript_path(data: dict[str, Any]) -> str: + """从 hook payload 中提取 transcript 路径字段。""" + return str(data.get("transcript_path") or "") + + +def read_stdin_json() -> dict: + """从 stdin 读取 JSON；失败时返回空字典。""" + import sys + try: + raw = sys.stdin.read() + if raw.strip(): + d = json.loads(raw) + if isinstance(d, dict): + return d + return {} + except Exception as e: + return {"_parse_error": str(e)} + + +def record_pre(pending_path: Path, data: dict) -> None: + """记录一条 PreToolUse，供后续 PostToolUse 计算耗时。""" + pending = st.load_pending(pending_path) + sid = data.get("session_id", "?") + tool = data.get("tool_name", "?") + key = f"{sid}::{tool}::{time.time_ns()}" + pending[key] = { + "start": time.time(), + "tool_name": tool, + "session_id": sid, + } + # 只保留最近 20 条 pending 记录 + if len(pending) > 20: + for k in list(pending.keys())[:-20]: + pending.pop(k, None) + st.save_pending(pending_path, pending) + + +def record_post(pending_path: Path, data: dict) -> int | None: + """把 PostToolUse 与之前的 PreToolUse 配对，并返回耗时毫秒数。""" + pending = st.load_pending(pending_path) + tool = data.get("tool_name") + sid = data.get("session_id", "?") + candidates = [(k, v) for k, v in pending.items() + if v.get("session_id") == sid and v.get("tool_name") == tool] + if not candidates: + return None + candidates.sort(key=lambda kv: kv[1]["start"]) + key, item = candidates[-1] + duration_ms = int((time.time() - item["start"]) * 1000) + pending.pop(key, None) + st.save_pending(pending_path, pending) + return duration_ms + + +def parse_transcript_tail( + path: str, + max_lines: int = 50, + last_offset: int = 0, +) -> dict[str, Any]: + """解析 transcript 的增量 JSONL 内容，提取 usage、model 与 tool 元数据。 + + 返回结构： + { + "tokens": {...} or None, # 最新累计 usage + "prev_tokens": {...} or None, # 倒数第二条累计 usage（兼容字段） + "model": str or None, + "offset": int, # 当前 EOF + "tool_records": [ + {"offset": int, "timestamp_ms": int | None, "tool": str, "call_id": str | None, "message_id": str | None, "kind": str}, + ... + ], + "usage_records": [ + {"offset": int, "tokens": {...}, "model": str | None, "message_id": str | None}, + ... + ], + } + + ``usage_records`` 会保留 ``last_offset`` 之后发现的全部 usage 项， + 这样调用方可以回放窗口内每一次模型请求，而不只是最后一个快照。 + """ + summary: dict[str, Any] = { + "tokens": None, + "prev_tokens": None, + "model": None, + "offset": 0, + "tool_records": [], + "usage_records": [], + } + if not path or not os.path.isfile(path): + return summary + + try: + file_size = os.path.getsize(path) + summary["offset"] = file_size + if file_size <= last_offset: + return summary + + with open(path, "rb") as fp: + read_start = max(0, int(last_offset or 0)) + fp.seek(read_start) + tail_bytes = fp.read() + except Exception: + return summary + + last_model_seen: str | None = None + usage_history: list[dict[str, Any]] = [] + pending_tool_records: list[dict[str, Any]] = [] + cursor = max(0, int(last_offset or 0)) + + for raw_line in tail_bytes.splitlines(keepends=True): + cursor += len(raw_line) + if not raw_line.lstrip().startswith(b"{"): + continue + try: + obj = json.loads(raw_line.decode("utf-8", errors="ignore")) + except Exception: + continue + + rec_type = str(obj.get("type") or "").strip().lower() if isinstance(obj, dict) else "" + message_id = _find_message_id(obj) + m = _find_model(obj) + if m: + last_model_seen = m + line_tool_records = _find_tool_records(obj) + if message_id and rec_type not in {"function_call", "function_call_result"}: + for pending_record in pending_tool_records: + pending_record["next_message_id"] = message_id + pending_tool_records = [] + for tool_record in line_tool_records: + tool_record["offset"] = cursor + tool_record["timestamp_ms"] = _find_timestamp_ms(obj) + tool_record["message_id"] = message_id + summary["tool_records"].append(tool_record) + pending_tool_records.append(tool_record) + usage = _find_usage(obj) + if not usage: + continue + + usage_history.append({ + "offset": cursor, + "tokens": usage, + "model": m or last_model_seen, + "message_id": message_id, + }) + if m: + summary["model"] = m + + if usage_history: + summary["usage_records"] = usage_history + summary["tokens"] = usage_history[-1]["tokens"] + if len(usage_history) >= 2: + summary["prev_tokens"] = usage_history[-2]["tokens"] + + if not summary.get("model") and last_model_seen: + summary["model"] = last_model_seen + return summary + + +def record_tool_usage( + state_path: Path, + sid: str, + data: dict, + skills_meta: dict, + rules_meta: dict, + active_agent: str | None = None, + collect_skills: bool = False, +) -> tuple[list[str], list[str]]: + """把一次 tool 调用命中的 skill/rule 写入 session state。 + + Returns: + (used_skills, used_rules): 本次工具调用命中的 skill/rule 名称列表。 + """ + tool = data.get("tool_name") + + def _update(state: dict[str, Any]) -> tuple[list[str], list[str]]: + sess = st.ensure_session(state, sid) + agent = active_agent or sess.get("current_agent") or "main" + + used_skills: set[str] = set() + used_rules: set[str] = set() + + # 1) 可选的 skill 收集 + if collect_skills: + # 直接 use_skill 调用 + direct = scanner.extract_skill_from_tool_call(data) + if direct: + st.bump_skill(sess, direct, via="use_skill", tool=tool, + meta=skills_meta.get(direct), agent=agent) + used_skills.add(direct) + + # 路径推断 + path_skills, _ = scanner.extract_paths_from_tool_call(data) + for s in path_skills: + st.bump_skill(sess, s, via="path-inferred", tool=tool, + meta=skills_meta.get(s), agent=agent) + used_skills.add(s) + + # 子模块命中 + for hit in scanner.extract_submodule_hits(data): + skill_name = hit["skill"] + # unknown 仅代表“命中子模块但无法确定 skill 名称”，不写入实时 skill 字段 + if not skill_name or skill_name == "unknown": + continue + st.bump_skill(sess, skill_name, via="submodule", tool=tool, + meta=skills_meta.get(skill_name), + submodule=hit["submodule"], agent=agent) + used_skills.add(skill_name) + + # Bash skill 脚本 + for s in scanner.extract_bash_skill_scripts(data): + st.bump_skill(sess, s, via="bash-script", tool=tool, + meta=skills_meta.get(s), agent=agent) + used_skills.add(s) + + # 2) 基于路径推断的 rule（始终开启） + _, path_rules = scanner.extract_paths_from_tool_call(data) + for r in path_rules: + st.bump_rule(sess, r, via="path-inferred", tool=tool, meta=rules_meta.get(r), agent=agent) + used_rules.add(r) + + # 3) 当前激活的 rule（始终开启） + for r in scanner.active_rules_for_call(data, rules_meta, active_agent=agent): + st.bump_rule(sess, r, via="active-rule", tool=tool, meta=rules_meta.get(r), agent=agent) + used_rules.add(r) + + st.prune_sessions(state) + return sorted(used_skills), sorted(used_rules) + + return st.update_state_locked(state_path, _update) + + + +def cache_inventory( + state_path: Path, + sid: str, + skills_meta: dict, + rules_meta: dict, +) -> None: + """把完整的 skill/rule inventory 缓存在 session state 中（在 SessionStart 调用）。""" + def _update(state: dict[str, Any]) -> None: + sess = st.ensure_session(state, sid) + sess["_skills_meta"] = skills_meta + sess["_rules_meta"] = rules_meta + sess["_inventory_scanned_at"] = time.time() + + # 为所有已知 skill/rule 预先初始化 count=0 的 usage 记录 + for name, meta in skills_meta.items(): + rec = sess["skills"].setdefault(name, { + "count": 0, "first_ts": None, "last_ts": None, + "tools": [], "via": ["static-scanned"], + "source": meta.get("source"), "version": meta.get("version"), + }) + if not rec.get("source"): + rec["source"] = meta.get("source") + if "static-scanned" not in rec.get("via", []): + rec.setdefault("via", []).append("static-scanned") + + for name, meta in rules_meta.items(): + rec = sess["rules"].setdefault(name, { + "count": 0, "first_ts": None, "last_ts": None, + "tools": [], "via": ["static-scanned"], "source": meta.get("source"), + }) + if "static-scanned" not in rec.get("via", []): + rec.setdefault("via", []).append("static-scanned") + if not rec.get("source"): + rec["source"] = meta.get("source") + + st.prune_sessions(state) + + st.update_state_locked(state_path, _update) + + +def load_cached_inventory(state_path: Path, sid: str) -> tuple[dict, dict]: + """从 state 中读取缓存的 skill/rule inventory，避免在热路径里执行 rglob。""" + state = st.load_state(state_path) + sess = state.get(sid, {}) + if not isinstance(sess, dict): + return {}, {} + return ( + sess.get("_skills_meta") or {}, + sess.get("_rules_meta") or {}, + ) + + +def get_session_usage(state_path: Path, sid: str) -> tuple[dict, dict]: + """返回某个 session 的 skill/rule usage 字典。""" + state = st.load_state(state_path) + sess = state.get(sid, {}) + if not isinstance(sess, dict): + return {}, {} + return sess.get("skills", {}) or {}, sess.get("rules", {}) or {} + +def related_transcript_paths(sid: str, transcript_path: str) -> list[str]: + """收集当前 session 的主 transcript 以及所有 subagent transcript。""" + paths: list[Path] = [] + current = Path(transcript_path).expanduser() if transcript_path else None + if current and current.is_file(): + paths.append(current) + + bundle_dir: Path | None = None + if current: + if current.name == f"{sid}.jsonl": + candidate = current.with_suffix("") + if candidate.is_dir(): + bundle_dir = candidate + else: + for parent in [current.parent, *current.parents]: + if parent.name == sid and parent.is_dir(): + bundle_dir = parent + break + + if bundle_dir is None and current: + candidate = current.parent / sid + if candidate.is_dir(): + bundle_dir = candidate + + if bundle_dir is not None: + main_transcript = bundle_dir.with_suffix(".jsonl") + if main_transcript.is_file(): + paths.append(main_transcript) + subagents_dir = bundle_dir / "subagents" + if subagents_dir.is_dir(): + paths.extend(sorted(path for path in subagents_dir.glob("*.jsonl") if path.is_file())) + + out: list[str] = [] + seen: set[str] = set() + for path in paths: + resolved = str(path.resolve()) + if resolved in seen: + continue + seen.add(resolved) + out.append(resolved) + return out + + +def _transcript_size_signature(paths: list[str]) -> tuple[tuple[str, int], ...]: + """基于路径和文件大小生成 transcript 稳定性签名。""" + signature: list[tuple[str, int]] = [] + for path in paths: + try: + size = Path(path).stat().st_size + except Exception: + size = -1 + signature.append((path, int(size))) + return tuple(signature) + + +def settled_related_transcript_paths( + sid: str, + transcript_path: str, + *, + max_wait_s: float = 2.0, + interval_s: float = 0.4, + stable_rounds: int = 2, +) -> list[str]: + """在 replay 前短暂等待 transcript 文件停止增长。""" + deadline = time.monotonic() + max(0.0, max_wait_s) + previous: tuple[tuple[str, int], ...] | None = None + stable_count = 0 + paths = related_transcript_paths(sid, transcript_path) + + while True: + paths = related_transcript_paths(sid, transcript_path) + current = _transcript_size_signature(paths) + if current == previous: + stable_count += 1 + else: + previous = current + stable_count = 0 + if stable_count >= stable_rounds or time.monotonic() >= deadline: + return paths + sleep_for = min(interval_s, max(0.0, deadline - time.monotonic())) + if sleep_for <= 0: + return paths + time.sleep(sleep_for) + + +def session_id_for_transcript_path(transcript_path: str) -> str | None: + """从 transcript 文件内容中反查 session id。""" + path = Path(transcript_path) + try: + with path.open("r", encoding="utf-8") as fp: + for _ in range(8): + line = fp.readline() + if not line: + break + try: + record = json.loads(line) + except Exception: + continue + session_id = record.get("sessionId") + if isinstance(session_id, str) and session_id.strip(): + return session_id.strip() + except Exception: + return None + return None + + +def resolve_transcript_path_alias(sid: str, transcript_path: str) -> str: + """把别名 transcript 路径解析成当前 session 下的真实文件路径。""" + if not transcript_path: + return "" + current = Path(transcript_path).expanduser() + if current.is_file(): + return str(current.resolve()) + + alias_session_id = current.stem if current.suffix == ".jsonl" else "" + if not alias_session_id or alias_session_id == sid: + return str(current) + + for candidate in related_transcript_paths(sid, transcript_path): + if session_id_for_transcript_path(candidate) == alias_session_id: + return candidate + return str(current) + +def compute_usage_delta(current: dict[str, int], prev: dict[str, int] | None) -> dict[str, int]: + """把一条 usage 记录归一化成当前要发出的 event payload。""" + _ = prev + out = dict(current) + out["total"] = int(out.get("input", 0) or 0) + int(out.get("output", 0) or 0) + return out + + +def normalize_tokens(tokens: dict[str, Any] | None) -> dict[str, int] | None: + """兼容多种 token 字段命名，并统一折叠成一套 schema。""" + if not isinstance(tokens, dict): + return None + + def _int(value: Any) -> int: + try: + return int(value or 0) + except Exception: + return 0 + + input_t = _int(tokens.get("input") if "input" in tokens else tokens.get("input_tokens")) + output_t = _int(tokens.get("output") if "output" in tokens else tokens.get("output_tokens")) + cache_read = _int(tokens.get("cache_read") if "cache_read" in tokens else tokens.get("cache_read_input_tokens")) + cache_creation = _int(tokens.get("cache_creation") if "cache_creation" in tokens else tokens.get("cache_creation_input_tokens")) + total = input_t + output_t + if input_t <= 0 and output_t <= 0 and total <= 0: + return None + out = {"input": input_t, "output": output_t, "cache_read": cache_read, "total": total} + if cache_creation > 0: + out["cache_creation"] = cache_creation + return out + + +def collect_transcript_entries( + state_path: Path, + sid: str, + transcript_path: str, + *, + max_lines: int, +) -> dict[str, Any]: + """读取 transcript 增量，并返回带 source offset 的解析结果。""" + state_data = st.load_state(state_path) + last_offset = st.get_transcript_offset(state_data, sid, transcript_path) + tail = parse_transcript_tail(transcript_path, max_lines=max_lines, last_offset=last_offset) + new_offset = int(tail.get("offset", 0) or 0) + + entries: list[dict[str, Any]] = [] + tool_records: list[dict[str, Any]] = [] + last_cumulative = st.get_last_cumulative_usage(state_data, sid, transcript_path) or None + usage_records = tail.get("usage_records") if isinstance(tail, dict) else None + if not isinstance(usage_records, list): + usage_records = [] + raw_tool_records = tail.get("tool_records") if isinstance(tail, dict) else None + if not isinstance(raw_tool_records, list): + raw_tool_records = [] + + for raw_entry in usage_records: + if not isinstance(raw_entry, dict): + continue + current = normalize_tokens(raw_entry.get("tokens")) + if not current: + continue + delta = compute_usage_delta(current, last_cumulative) + entries.append({ + "offset": int(raw_entry.get("offset", 0) or 0), + "tokens": delta, + "model": str(raw_entry.get("model") or tail.get("model") or "") or None, + "message_id": str(raw_entry.get("message_id") or "").strip() or None, + }) + last_cumulative = current + + for raw_tool_record in raw_tool_records: + if not isinstance(raw_tool_record, dict): + continue + tool_name = str(raw_tool_record.get("tool") or "").strip() + if not tool_name: + continue + tool_records.append({ + "offset": int(raw_tool_record.get("offset", 0) or 0), + "timestamp_ms": int(raw_tool_record.get("timestamp_ms", 0) or 0) or None, + "tool": tool_name, + "call_id": str(raw_tool_record.get("call_id") or "").strip() or None, + "kind": str(raw_tool_record.get("kind") or "").strip() or None, + "message_id": str(raw_tool_record.get("message_id") or "").strip() or None, + "next_message_id": str(raw_tool_record.get("next_message_id") or "").strip() or None, + "arguments": raw_tool_record.get("arguments"), + "arguments_display_text": raw_tool_record.get("arguments_display_text"), + "result_content": raw_tool_record.get("result_content"), + "raw_response": raw_tool_record.get("raw_response"), + "output_text": raw_tool_record.get("output_text"), + }) + + return { + "entries": entries, + "tool_records": tool_records, + "new_offset": new_offset, + "last_cumulative": last_cumulative, + } + + +def find_current_tool_message_id( + state_path: Path, + sid: str, + transcript_path: str, + tool_name: str, + *, + max_lines: int = 80, +) -> str | None: + """找到当前 tool 事件应该归属的 message id。""" + if not transcript_path or not tool_name: + return None + scan = collect_transcript_entries( + state_path=state_path, + sid=sid, + transcript_path=transcript_path, + max_lines=max_lines, + ) + tool_records = scan.get("tool_records") or [] + latest_message_id: str | None = None + latest_offset = -1 + for rec in tool_records: + if not isinstance(rec, dict): + continue + if str(rec.get("tool") or "") != str(tool_name): + continue + message_id = ( + str(rec.get("next_message_id") or "").strip() + or str(rec.get("message_id") or "").strip() + or None + ) + if not message_id: + continue + offset = int(rec.get("offset", 0) or 0) + if offset >= latest_offset: + latest_offset = offset + latest_message_id = message_id + return latest_message_id + + +def tool_details_from_record(rec: dict[str, Any]) -> dict[str, Any]: + """把原始 tool 记录投影成日志里使用的紧凑 detail 结构。""" + details: dict[str, Any] = {} + call_id = str(rec.get("call_id") or "").strip() or None + if call_id: + details["call_id"] = call_id + arguments = rec.get("arguments") + if arguments is not None: + details["arguments"] = arguments + arguments_display_text = rec.get("arguments_display_text") + if arguments_display_text is not None: + details["arguments_display_text"] = arguments_display_text + result_content = rec.get("result_content") + if result_content is not None: + details["result_content"] = result_content + raw_response = rec.get("raw_response") + if isinstance(raw_response, dict) and raw_response: + details["raw_response"] = raw_response + output_text = rec.get("output_text") + if output_text is not None: + details["output_text"] = output_text + next_message_id = str(rec.get("next_message_id") or "").strip() or None + original_message_id = str(rec.get("message_id") or "").strip() or None + if next_message_id and original_message_id: + details["original_message_id"] = original_message_id + if next_message_id: + details["next_message_id"] = next_message_id + if original_message_id and next_message_id and original_message_id != next_message_id: + details["message_id_reassigned"] = True + return details + + +def tool_record_merge_key(rec: dict[str, Any]) -> str: + """为一条 tool record 生成去重合并时使用的稳定 key。""" + call_id = str(rec.get("call_id") or "").strip() + if call_id: + return f"call_id:{call_id}" + offset = int(rec.get("offset", 0) or 0) + tool = str(rec.get("tool") or "").strip() + message_id = str(rec.get("message_id") or "").strip() + timestamp_ms = int(rec.get("timestamp_ms", 0) or 0) + return f"fallback:{tool}:{message_id}:{timestamp_ms}:{offset}" + + +def merge_tool_records(tool_records: list[dict[str, Any]]) -> list[dict[str, Any]]: + """对指向同一次调用的 tool 记录做去重合并。""" + merged: dict[str, dict[str, Any]] = {} + order: list[str] = [] + for rec in tool_records: + if not isinstance(rec, dict): + continue + key = tool_record_merge_key(rec) + if key not in merged: + merged[key] = { + "offset": int(rec.get("offset", 0) or 0), + "timestamp_ms": int(rec.get("timestamp_ms", 0) or 0) or None, + "tool": str(rec.get("tool") or "").strip(), + "call_id": str(rec.get("call_id") or "").strip() or None, + "kind": str(rec.get("kind") or "").strip() or None, + "message_id": str(rec.get("message_id") or "").strip() or None, + "next_message_id": str(rec.get("next_message_id") or "").strip() or None, + "arguments": rec.get("arguments"), + "arguments_display_text": rec.get("arguments_display_text"), + "result_content": rec.get("result_content"), + "raw_response": rec.get("raw_response"), + "output_text": rec.get("output_text"), + } + order.append(key) + continue + + current = merged[key] + current["offset"] = max(int(current.get("offset", 0) or 0), int(rec.get("offset", 0) or 0)) + current_ts = int(current.get("timestamp_ms", 0) or 0) + rec_ts = int(rec.get("timestamp_ms", 0) or 0) + if current_ts <= 0 and rec_ts > 0: + current["timestamp_ms"] = rec_ts + for field in ("tool", "call_id", "kind", "message_id", "next_message_id", "arguments", "arguments_display_text", "result_content", "raw_response", "output_text"): + if current.get(field) is None and rec.get(field) is not None: + current[field] = rec.get(field) + if rec.get("arguments") is not None: + current["arguments"] = rec.get("arguments") + if rec.get("arguments_display_text") is not None: + current["arguments_display_text"] = rec.get("arguments_display_text") + if rec.get("result_content") is not None: + current["result_content"] = rec.get("result_content") + if rec.get("raw_response") is not None: + current["raw_response"] = rec.get("raw_response") + if rec.get("output_text") is not None: + current["output_text"] = rec.get("output_text") + + return [merged[key] for key in order] + + +def tool_context_claim_key(rec: dict[str, Any]) -> str: + """为 tool context 生成 claim 去重键，避免重复消费。""" + call_id = str(rec.get("call_id") or "").strip() + if call_id: + return f"tool_call|{call_id}" + tool = str(rec.get("tool") or "").strip() + message_id = str(rec.get("message_id") or "").strip() + timestamp_ms = int(rec.get("timestamp_ms", 0) or 0) + offset = int(rec.get("offset", 0) or 0) + return f"tool_call|{tool}|{message_id}|{timestamp_ms}|{offset}" + + +def find_current_tool_context( + state_path: Path, + sid: str, + transcript_path: str, + tool_name: str, + *, + call_id: str | None = None, + event_ts: float | None = None, + max_lines: int = 80, + claim: bool = False, +) -> dict[str, Any] | None: + """为当前 tool 调用解析最合适的 transcript 上下文块。""" + if not tool_name: + return None + event_ts_ms = int(float(event_ts) * 1000) if event_ts else None + normalized_call_id = str(call_id or "").strip() or None + + resolved_path = resolve_transcript_path_alias(sid, transcript_path) + candidate_paths: list[str] = [] + if resolved_path: + candidate_paths.append(resolved_path) + for path in related_transcript_paths(sid, transcript_path): + if path not in candidate_paths: + candidate_paths.append(path) + + candidates: list[tuple[tuple[int, int, int], dict[str, Any]]] = [] + for path in candidate_paths: + scan = collect_transcript_entries( + state_path=state_path, + sid=sid, + transcript_path=path, + max_lines=max_lines, + ) + tool_records = merge_tool_records(scan.get("tool_records") or []) + for rec in tool_records: + if not isinstance(rec, dict): + continue + if str(rec.get("tool") or "") != str(tool_name): + continue + rec_call_id = str(rec.get("call_id") or "").strip() or None + if normalized_call_id and rec_call_id != normalized_call_id: + continue + message_id = ( + str(rec.get("next_message_id") or "").strip() + or str(rec.get("message_id") or "").strip() + or None + ) + if not message_id: + continue + offset = int(rec.get("offset", 0) or 0) + timestamp_ms = int(rec.get("timestamp_ms", 0) or 0) + if normalized_call_id: + score = (0, 0, -offset) + elif event_ts_ms and timestamp_ms > 0: + delta = timestamp_ms - event_ts_ms + score = (abs(delta), 0 if delta >= 0 else 1, -offset) + else: + score = (10**12, 1, -offset) + candidates.append(( + score, + { + "message_id": message_id, + "transcript_path": path, + "agent": agent_identity.agent_for_transcript_path(path, "main"), + "pt_id": agent_identity.pt_id_from_transcript_path(path), + "tool_details": tool_details_from_record(rec), + "_claim_key": tool_context_claim_key(rec), + }, + )) + + candidates.sort(key=lambda item: item[0]) + saw_duplicate = False + for _, candidate in candidates: + if not claim: + out = { + "message_id": str(candidate.get("message_id") or ""), + "transcript_path": str(candidate.get("transcript_path") or ""), + "agent": str(candidate.get("agent") or "main"), + "tool_details": dict(candidate.get("tool_details") or {}), + } + pt_id = candidate.get("pt_id") + if pt_id: + out["pt_id"] = pt_id + return out + if st.claim_transcript_event( + state_path, + sid, + str(candidate.get("_claim_key") or ""), + str(candidate.get("transcript_path") or "") or None, + ): + out = { + "message_id": str(candidate.get("message_id") or ""), + "transcript_path": str(candidate.get("transcript_path") or ""), + "agent": str(candidate.get("agent") or "main"), + "tool_details": dict(candidate.get("tool_details") or {}), + } + pt_id = candidate.get("pt_id") + if pt_id: + out["pt_id"] = pt_id + return out + saw_duplicate = True + + if saw_duplicate: + return {"duplicate": True} + return None + + +def extract_tool_call_id(data: dict[str, Any]) -> str | None: + """从事件 payload 或 tool_input 中提取 tool call id。""" + for key in ("call_id", "callId", "tool_call_id", "toolCallId"): + value = data.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + + tool_input = data.get("tool_input") + if isinstance(tool_input, dict): + for key in ("call_id", "callId", "tool_call_id", "toolCallId", "id"): + value = tool_input.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + return None + + +def find_fallback_usage_event( + tool_event: dict[str, Any], + usage_events: list[dict[str, Any]], +) -> dict[str, Any] | None: + """当找不到直接匹配的 tool usage 时，为 AgentLens 选一条兜底 usage。""" + tool_name = str(tool_event.get("tool") or "").strip() + transcript_path = str(tool_event.get("transcript_path") or "").strip() + agent = str(tool_event.get("agent") or "main").strip() or "main" + + strong_matches: list[dict[str, Any]] = [] + weak_matches: list[dict[str, Any]] = [] + for usage_event in usage_events: + if not isinstance(usage_event, dict): + continue + if (str(usage_event.get("agent") or "main").strip() or "main") != agent: + continue + if transcript_path and str(usage_event.get("transcript_path") or "").strip() != transcript_path: + continue + if not str(usage_event.get("message_id") or "").strip(): + continue + + weak_matches.append(usage_event) + event_tool = str(usage_event.get("tool") or "").strip() + if tool_name and event_tool in {tool_name, "model_request"}: + strong_matches.append(usage_event) + + if strong_matches: + return strong_matches[-1] + if weak_matches: + return weak_matches[-1] + return None + + +def build_transcript_event_key(*, kind: str, tool: str, entry: dict[str, Any]) -> str: + """为 transcript 重放出的事件生成幂等键。""" + _ = (kind, tool) + return "|".join(["model_request", str(entry.get("offset", 0) or 0)]) + +def _find_usage(obj: Any, _depth: int = 0) -> dict | None: + """递归查找 JSON 对象里的 token usage。""" + if _depth > 10: + return None + if isinstance(obj, dict): + if "usage" in obj and isinstance(obj["usage"], dict): + u = obj["usage"] + picked = {k: u[k] for k in TOKEN_KEYS if k in u} + if picked: + return picked + for v in obj.values(): + r = _find_usage(v, _depth + 1) + if r: + return r + elif isinstance(obj, list): + for v in obj: + r = _find_usage(v, _depth + 1) + if r: + return r + return None + + +def _find_model(obj: Any, _depth: int = 0) -> str | None: + """递归查找 JSON 对象里的非空模型名。 + + 优先使用 `providerData.model` 或顶层 `model`， + 其次回退到 `requestModelName` / `requestModelId`。 + """ + if _depth > 10: + return None + if isinstance(obj, dict): + for k in MODEL_KEYS: + v = obj.get(k) + if isinstance(v, str) and v.strip(): + return v.strip() + for v in obj.values(): + r = _find_model(v, _depth + 1) + if r: + return r + elif isinstance(obj, list): + for v in obj: + r = _find_model(v, _depth + 1) + if r: + return r + return None + + +def _find_message_id(obj: Any, _depth: int = 0) -> str | None: + """递归查找稳定的 message/request 分组标识。""" + if _depth > 10: + return None + if isinstance(obj, dict): + provider = obj.get("providerData") + if isinstance(provider, dict): + for key in MESSAGE_ID_KEYS: + value = provider.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + extra = obj.get("extra") + if isinstance(extra, dict): + for key in MESSAGE_ID_KEYS: + value = extra.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + for key in MESSAGE_ID_KEYS: + value = obj.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + for v in obj.values(): + r = _find_message_id(v, _depth + 1) + if r: + return r + elif isinstance(obj, list): + for v in obj: + r = _find_message_id(v, _depth + 1) + if r: + return r + return None + + +def _find_timestamp_ms(obj: Any, _depth: int = 0) -> int | None: + """递归查找 transcript 中的毫秒级时间戳。""" + if _depth > 10: + return None + if isinstance(obj, dict): + value = obj.get("timestamp") + if isinstance(value, (int, float)) and value > 0: + return int(value) + provider = obj.get("providerData") + if isinstance(provider, dict): + value = provider.get("timestamp") + if isinstance(value, (int, float)) and value > 0: + return int(value) + for v in obj.values(): + r = _find_timestamp_ms(v, _depth + 1) + if r: + return r + elif isinstance(obj, list): + for v in obj: + r = _find_timestamp_ms(v, _depth + 1) + if r: + return r + return None + + +def _find_tool_records(obj: Any) -> list[dict[str, Any]]: + """从 transcript 记录里提取 tool 调用与 tool 结果片段。""" + out: list[dict[str, Any]] = [] + if not isinstance(obj, dict): + return out + + rec_type = str(obj.get("type") or "").strip().lower() + provider = obj.get("providerData") if isinstance(obj.get("providerData"), dict) else {} + if rec_type in {"function_call", "function_call_result"}: + tool_name = str(obj.get("name") or "").strip() + if tool_name: + record = { + "tool": tool_name, + "call_id": str(obj.get("callId") or "").strip() or None, + "kind": rec_type, + } + if rec_type == "function_call": + arguments = obj.get("arguments") + if isinstance(arguments, str) and arguments.strip(): + record["arguments"] = arguments + arguments_display_text = provider.get("argumentsDisplayText") + if isinstance(arguments_display_text, str) and arguments_display_text.strip(): + record["arguments_display_text"] = arguments_display_text.strip() + if rec_type == "function_call_result": + tool_result = provider.get("toolResult") + if isinstance(tool_result, dict): + content = tool_result.get("content") + if content is not None: + record["result_content"] = content + raw_response = tool_result.get("rawResponse") + if isinstance(raw_response, dict): + record["raw_response"] = raw_response + output = obj.get("output") + if isinstance(output, dict): + output_text = output.get("text") + if output_text is not None: + record["output_text"] = output_text + out.append(record) + + content = obj.get("content") + if isinstance(content, list): + for item in content: + if not isinstance(item, dict): + continue + if str(item.get("type") or "").strip().lower() != "tool_use": + continue + tool_name = str(item.get("name") or "").strip() + if not tool_name: + continue + out.append({ + "tool": tool_name, + "call_id": str(item.get("callId") or "").strip() or None, + "kind": "tool_use", + }) + + return out diff --git a/.codebuddy/skills/agent-observability/scripts/core/devflow.py b/.codebuddy/skills/agent-observability/scripts/core/devflow.py new file mode 100644 index 0000000..ab7ef79 --- /dev/null +++ b/.codebuddy/skills/agent-observability/scripts/core/devflow.py @@ -0,0 +1,234 @@ +"""Devflow 感知层（可选增强，不属于通用采集核心）。 + +这一层只在项目实际跑着 `.codebuddy/runtime` 描述的 multi-agents-devflow 工作流时才生效： +- 判断当前 session 是否属于某个 devflow team（`multi-agents-devflow-{task_slug}`） +- 读取该 team 对应的 `workflow-state.json`，投影成精简 stage 快照 +- 和上一次观测到的快照 diff，只把真正变化的 stage 产出为事件 + +任何解析失败（team 目录不存在 / workflow-state.json 缺失或损坏 / 字段缺失）都必须 +优雅降级为 None / 空列表，绝不能让不跑 devflow 的普通项目因为这一层报错。 +""" +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from . import agent_identity, state as st + + +def task_slug_from_team_dir(team_dir: Path) -> str | None: + """从 devflow team 目录名里剥离出 task_slug（固定前缀 `multi-agents-devflow-`）。""" + name = team_dir.name + prefix = agent_identity.TEAM_PREFIX + if not name.startswith(prefix): + return None + slug = name[len(prefix):].strip() + return slug or None + + +def resolve_devflow_context(cwd: str, sid: str, cached_team_dir: str | None = None) -> dict[str, Any] | None: + """判断当前 session 是否处于某次 devflow 运行中；不是则返回 None。 + + 优先复用 `agent_identity.resolve_team_dir` 做 team 发现（Classic 全部场景，以及 + Portable 在 `topology: team` 宿主——目前是 CodeBuddy——下也走同一套 + `multi-agents-devflow-{task_slug}` 命名，可以直接复用，不用区分 edition）。 + + `topology: spawn` 的宿主（Codex/Claude/Cursor 的 Portable 适配器）不创建 team + 目录，找不到时退化成 `_scan_artifacts_for_active_run` 直接扫 `artifacts/` 目录。 + 这个兜底本身就是尽力而为的启发式，不保证唯一/精确，见该函数的说明。 + + `artifacts_dir` 按 `devflow.defaults.yaml` 的默认值 `{project_root}/artifacts/{task_slug}` + 推算——项目若覆写了 `artifacts.root_dir`，这里暂不感知，读取 workflow-state.json + 找不到文件会安全返回 None，不会误报。 + """ + team_dir = agent_identity.resolve_team_dir(cwd, sid, cached_team_dir) + if team_dir is not None: + task_slug = task_slug_from_team_dir(team_dir) + if task_slug: + artifacts_dir = Path(cwd).expanduser() / "artifacts" / task_slug + workflow_state_path = artifacts_dir / "workflow-state.json" + return { + "task_slug": task_slug, + "team_dir": str(team_dir), + "artifacts_dir": str(artifacts_dir), + "workflow_state_path": str(workflow_state_path), + } + return _scan_artifacts_for_active_run(cwd) + + +def _scan_artifacts_for_active_run(cwd: str) -> dict[str, Any] | None: + """没有 team 目录时的兜底发现：直接扫 `{cwd}/artifacts/*/workflow-state.json`。 + + 用于 `topology: spawn` 的宿主（没有 `.codebuddy/teams/` 这类目录可以反查），以及 + Classic 没有真正暴露 `team_create` 时的降级场景。这是启发式，不是精确匹配： + 多个 task_slug 存在时，优先选"还没跑完"的那个；都跑完或都没跑完时选 + `workflow-state.json` 文件 mtime 最新的一个。"跑完"的判定要兼容两套 schema—— + Classic（v1.3）没有顶层 `status` 字段，看 `last_event == "workflow_completed"`； + Portable（v2.0）看顶层 `status in {"completed", "failed"}`。项目里如果同时有多个 + 真正并发、都还没跑完的 devflow 运行，这个兜底可能选错——已知限制，不在这次范围内解决。 + """ + artifacts_root = Path(cwd).expanduser() / "artifacts" + if not artifacts_root.is_dir(): + return None + candidates: list[tuple[tuple[int, float], str, Path]] = [] + try: + children = list(artifacts_root.iterdir()) + except Exception: + return None + for child in children: + if not child.is_dir(): + continue + state_path = child / "workflow-state.json" + state = read_workflow_state(str(state_path)) + if not isinstance(state, dict) or not isinstance(state.get("stages"), dict): + continue + try: + mtime = state_path.stat().st_mtime + except Exception: + mtime = 0.0 + finished = ( + str(state.get("last_event") or "") == "workflow_completed" + or str(state.get("status") or "") in {"completed", "failed"} + ) + not_finished = 0 if finished else 1 + candidates.append(((not_finished, mtime), child.name, state_path)) + if not candidates: + return None + candidates.sort(key=lambda item: item[0], reverse=True) + _, task_slug, state_path = candidates[0] + return { + "task_slug": task_slug, + "team_dir": None, + "artifacts_dir": str(state_path.parent), + "workflow_state_path": str(state_path), + } + + +def read_workflow_state(path: str) -> dict[str, Any] | None: + """安全读取 workflow-state.json；文件不存在或解析失败都返回 None。""" + try: + p = Path(path) + if not p.is_file(): + return None + data = json.loads(p.read_text("utf-8")) + return data if isinstance(data, dict) else None + except Exception: + return None + + +def stage_snapshot(workflow_state: dict[str, Any] | None) -> dict[str, Any]: + """把 workflow-state.json 投影成精简快照，供比对和事件输出使用。 + + 兼容两套 schema：Classic（v1.3，字段名 `executor`，有 `review_result`）和 + Portable（v2.0，字段名 `executor_role`，没有 `review_result`，但顶层多了 + `execution_mode`/`host_adapter`/`run_id`/`team_name` 这些 Classic 没有的上下文）。 + 按 `version` 字段区分，取不到就都尝试取一遍，不强制要求调用方先判断是哪个 edition。 + """ + if not isinstance(workflow_state, dict): + return {} + stages_raw = workflow_state.get("stages") + stages: dict[str, Any] = {} + if isinstance(stages_raw, dict): + for name, info in stages_raw.items(): + if not isinstance(info, dict): + continue + stages[name] = { + "status": info.get("status"), + "executor": info.get("executor") or info.get("executor_role"), + "retry_count": info.get("retry_count", 0), + "review_result": info.get("review_result"), + } + return { + "current_stage": workflow_state.get("current_stage"), + "size_class": workflow_state.get("size_class"), + "run_mode": workflow_state.get("run_mode"), + "schema_version": workflow_state.get("version"), + "execution_mode": workflow_state.get("execution_mode"), + "host_adapter": workflow_state.get("host_adapter"), + "run_id": workflow_state.get("run_id"), + "stages": stages, + } + + +def diff_stage_changes(prev: dict[str, Any] | None, curr: dict[str, Any]) -> list[dict[str, Any]]: + """比较两次 stage 快照，只返回 status/retry_count/review_result 真正变化的阶段。 + + 首次观测（`prev` 为 None，即这个 session 第一次检测到 devflow）不产出任何变更—— + 避免刚接入 observability 时，把一个已经跑了大半的 devflow 运行的全部历史阶段 + 当成"新事件"一次性炸出来。之后每次变化都会被正常捕获。 + """ + if not isinstance(curr, dict) or prev is None: + return [] + curr_stages = curr.get("stages") or {} + prev_stages = prev.get("stages") if isinstance(prev, dict) else {} + if not isinstance(prev_stages, dict): + prev_stages = {} + changes: list[dict[str, Any]] = [] + for name, info in curr_stages.items(): + if not isinstance(info, dict): + continue + before = prev_stages.get(name) + before = before if isinstance(before, dict) else {} + fields = ("status", "retry_count", "review_result") + if any(before.get(f) != info.get(f) for f in fields): + changes.append({ + "stage": name, + "status": info.get("status"), + "executor": info.get("executor"), + "retry_count": info.get("retry_count", 0), + "review_result": info.get("review_result"), + }) + return changes + + +def resolve_and_diff(state_path: Path, sid: str, cwd: str) -> dict[str, Any] | None: + """解析当前 session 归属的 devflow 上下文，返回上下文 + 本次观测到的 stage 变更。 + + **不缓存"当前归属哪个 task_slug"这个结论本身**——同一个 sid 在其生命周期内可能 + 先后归属不同的 devflow 运行。这不是理论场景：当运行时没有暴露真正的 + `team_create`/`send_message`（因此没有独立的 team 成员 sid），devflow 会退化成 + 在同一个长生命周期 session 里，先后跑好几次 `/start-devflow`；每次都必须重新判定 + "现在最合适的 task_slug 是哪个"，而不能沿用第一次探测到的那个——沿用旧结论会把 + 第二次运行的所有事件都错误地归到第一次的 task_slug 下。 + + 重新判定的代价很低：`resolve_devflow_context` 对团队目录的探测会用上一次找到的 + `team_dir` 做快速校验（只在真正失效时才重新扫描 `.codebuddy/teams/`）； + `artifacts/` 兜底扫描也只是一次浅层 `iterdir()`，不是全量遍历。 + + 真正跨调用持久化的只有**按 task_slug 分别保存的 stage 快照历史**——切换到另一个 + task_slug 不会污染对方的 diff 基线，也不会因为切回旧 task_slug 而把它已经观测过的 + 历史重新当成"首次观测"。 + + 返回 None 表示这次调用没有探测到任何 devflow 上下文；否则返回 + `{task_slug, artifacts_dir, workflow_state_path, current_stage, size_class, changes}`。 + """ + + def _update(state: dict[str, Any]) -> dict[str, Any] | None: + dv = st.get_devflow_state(state, sid) + cached_team_dir = (dv.get("context") or {}).get("team_dir") if isinstance(dv.get("context"), dict) else None + context = resolve_devflow_context(cwd, sid, cached_team_dir) + if not isinstance(context, dict): + st.set_devflow_state(state, sid, {"context": None}) + return None + + task_slug = str(context.get("task_slug") or "") + workflow_state = read_workflow_state(context.get("workflow_state_path", "")) + curr = stage_snapshot(workflow_state) + snapshots = dv.get("snapshots") + if not isinstance(snapshots, dict): + snapshots = {} + prev = snapshots.get(task_slug) + changes = diff_stage_changes(prev if isinstance(prev, dict) else None, curr) + snapshots[task_slug] = curr + st.set_devflow_state(state, sid, {"context": context, "snapshots": snapshots}) + + result = dict(context) + result["current_stage"] = curr.get("current_stage") + result["size_class"] = curr.get("size_class") + result["schema_version"] = curr.get("schema_version") + result["execution_mode"] = curr.get("execution_mode") + result["changes"] = changes + return result + + return st.update_state_locked(state_path, _update) diff --git a/.codebuddy/skills/agent-observability/scripts/core/emitter.py b/.codebuddy/skills/agent-observability/scripts/core/emitter.py new file mode 100644 index 0000000..a41dfc0 --- /dev/null +++ b/.codebuddy/skills/agent-observability/scripts/core/emitter.py @@ -0,0 +1,578 @@ +"""ndjson 发射层。 + +只输出当前真正需要的核心字段： +- common: data_source +- tool: event, sid, ts, agent, tool, ms, skill, rule, transcript_path +- usage: event, sid, ts, agent, tool, tokens, model, cost_usd, transcript_path +- start: event, sid, ts, agent +- stop: event, sid, ts, agent, tokens, model, cost_usd, cost_session_usd, transcript_path +- error: event, sid, ts, phase, error + +历史 x_* 扩展字段、stop skill/rule 汇总、schema 版本号等都不再生成。 + +`cost_usd` 的价格表 = 内置 `config/pricing.json` 与用户覆盖文件（默认 +`/.codebuddy/agent-observability/pricing.overrides.json`，可用 +`AOBS_PRICING_OVERRIDES_PATH` 改路径）的字段级合并结果，详见 `load_prices()`。 +""" +from __future__ import annotations + +import json +import os +import time +from functools import lru_cache +from pathlib import Path +from typing import Any + +from . import cls_sink +from . import state as st + +DEFAULT_PRICES_PATH = Path(__file__).resolve().parents[2] / "config" / "pricing.json" +DEFAULT_DATA_SOURCE = "codebuddy-cli" + + +def _default_overrides_path() -> Path | None: + """用户定价覆盖文件的默认位置：`/.codebuddy/agent-observability/pricing.overrides.json`。 + + 刻意放在 `skills/` 树**外**：`scripts/build-classic-hosts.py` 会整棵同步 + `.codebuddy/skills`，放树内会 (a) 每次改动都产生 `--check` drift，(b) 把用户的 + 自定义价格复制进 `.claude`/`.cursor` 宿主包；放树外则既不参与同步，也不会被 + `config/pricing.json` 的后续更新冲掉。 + """ + try: + project_root = Path(__file__).resolve().parents[5] + except IndexError: + return None + return project_root / ".codebuddy" / "agent-observability" / "pricing.overrides.json" + + +# None 表示"拿不到项目根"，此时等同于"没有覆盖文件"（纯内置价格表）。 +DEFAULT_OVERRIDES_PATH: Path | None = _default_overrides_path() + + +def get_log_path(base_dir: Path) -> Path: + """返回 metrics 日志路径，并确保目录已存在。""" + log_dir = base_dir / "logs" + log_dir.mkdir(parents=True, exist_ok=True) + return log_dir / "metrics.ndjson" + + +def _first_record_ts(log_path: Path, sid: str) -> float | None: + """从 metrics.ndjson 中找到某个 session 的最早 ts。 + + state 只保留最近几个 session，长流程或历史 session 可能被 prune 后重建， + 因此 `started_at` 不能作为唯一权威。日志里的首条 ts 更接近 + session/workflow 的真实起点。 + """ + if not sid or not log_path.exists(): + return None + first_ts: float | None = None + try: + with log_path.open("r", encoding="utf-8") as fp: + for line in fp: + if not line.strip().startswith("{"): + continue + try: + rec = json.loads(line) + except Exception: + continue + if rec.get("sid") != sid: + continue + try: + ts = float(rec.get("ts")) + except Exception: + continue + if first_ts is None or ts < first_ts: + first_ts = ts + except Exception: + return None + return first_ts + + +def _inject_session_duration(record: dict[str, Any], log_path: Path) -> None: + """从 state 文件读取 session started_at，计算 session_duration_sec 注入 record。 + 同时扫描 metrics.ndjson 累计 session 级别 token 总量。""" + sid = record.get("sid") + if not sid or "session_duration_sec" in record: + return + try: + now = float(record.get("ts") or time.time()) + started_candidates: list[float] = [] + state_path = log_path.parent / ".state.json" + state = st.load_state(state_path) + sess = state.get(sid) + if isinstance(sess, dict): + try: + state_started_at = float(sess.get("started_at") or 0) + except Exception: + state_started_at = 0.0 + if state_started_at > 0: + started_candidates.append(state_started_at) + log_started_at = _first_record_ts(log_path, str(sid)) + if log_started_at is not None and log_started_at > 0: + started_candidates.append(log_started_at) + if not started_candidates: + started_candidates.append(now) + started_at = min(started_candidates) + record["session_duration_sec"] = round(max(0.0, now - started_at), 3) + except Exception: + return + + # 累计 session 级别 token（扫描 metrics.ndjson 尾部） + try: + total_input = 0 + total_output = 0 + total_cache = 0 + if log_path.exists(): + with log_path.open("r", encoding="utf-8") as fp: + for line in fp: + if not line.strip().startswith("{"): + continue + try: + rec = json.loads(line) + except Exception: + continue + if rec.get("sid") != sid or rec.get("event") != "usage": + continue + tokens = rec.get("tokens") + if not isinstance(tokens, dict): + continue + total_input += int(tokens.get("input") or 0) + total_output += int(tokens.get("output") or 0) + total_cache += int(tokens.get("cache_read") or 0) + record["session_total_tokens"] = total_input + total_output + record["session_input_tokens"] = total_input + record["session_output_tokens"] = total_output + record["session_cache_tokens"] = total_cache + except Exception: + return + + +def emit(log_path: Path, record: dict[str, Any]) -> None: + """向 ndjson 日志追加一条记录。""" + record.setdefault("data_source", DEFAULT_DATA_SOURCE) + record.setdefault("ts", time.time()) + _inject_session_duration(record, log_path) + with log_path.open("a", encoding="utf-8") as fp: + fp.write(json.dumps(record, ensure_ascii=False) + "\n") + try: + cls_sink.mirror_record(record) + except Exception: + return + + +def build_tool_event( + sid: str, + tool: str | None, + duration_ms: int | None, + active_agent: str | None = None, + transcript_path: str | None = None, + turn_id: str | None = None, + task_slug: str | None = None, + stage: str | None = None, + pt_id: str | None = None, +) -> dict[str, Any]: + """构造 tool 事件记录，只包含调用信息与耗时。 + + `task_slug` / `stage` / `pt_id` 只有在检测到 devflow 运行时才会出现（见 + `core/devflow.py`），非 devflow 项目不受影响。 + """ + record = { + "event": "tool", + "sid": sid, + "agent": active_agent or "main", + "tool": tool, + "ms": duration_ms, + } + if transcript_path: + record["transcript_path"] = transcript_path + if turn_id: + record["turn_id"] = turn_id + if task_slug: + record["task_slug"] = task_slug + if stage: + record["stage"] = stage + if pt_id: + record["pt_id"] = pt_id + return record + + +def build_usage_event( + sid: str, + tool: str | None, + tokens: dict[str, Any], + active_agent: str | None = None, + model: str | None = None, + cost_usd: float | None = None, + transcript_path: str | None = None, + source_offset: int | None = None, + turn_id: str | None = None, + message_id: str | None = None, + task_slug: str | None = None, + stage: str | None = None, +) -> dict[str, Any]: + """构造 LLM usage 事件记录。""" + record: dict[str, Any] = { + "event": "usage", + "sid": sid, + "agent": active_agent or "main", + "tool": tool, + "tokens": tokens, + } + if model: + record["model"] = model + if cost_usd is not None: + record["cost_usd"] = cost_usd + if transcript_path: + record["transcript_path"] = transcript_path + if source_offset is not None: + record["source_offset"] = source_offset + if turn_id: + record["turn_id"] = turn_id + if message_id: + record["message_id"] = message_id + if task_slug: + record["task_slug"] = task_slug + if stage: + record["stage"] = stage + return record + + +def build_start_event( + sid: str, + agent: str | None = None, +) -> dict[str, Any]: + """构造最小化的 start 事件记录。 + + 初始化阶段没有明确角色时，统一记为 ``main``。 + """ + return { + "event": "start", + "sid": sid, + "agent": agent or "main", + } + + +def build_prompt_submit_event( + sid: str, + turn_id: str, + agent: str | None = None, + prompt_len: int | None = None, +) -> dict[str, Any]: + """构造 `user_prompt_submit` 事件，作为 turn 边界标记。 + + 这条记录会落到 metrics.ndjson，方便下游把 turn 和 AgentLens trace 对上。 + """ + record: dict[str, Any] = { + "event": "user_prompt_submit", + "sid": sid, + "agent": agent or "main", + "turn_id": turn_id, + } + if prompt_len is not None: + record["prompt_len"] = int(prompt_len) + return record + + +def build_stop_event( + sid: str, + tokens: dict | None = None, + model: str | None = None, + cost_usd: float | None = None, + cost_session_usd: float | None = None, + active_agent: str | None = None, + transcript_path: str | None = None, + source_offset: int | None = None, + turn_id: str | None = None, + message_id: str | None = None, + task_slug: str | None = None, + stage: str | None = None, +) -> dict[str, Any]: + """构造最小化的 stop 事件记录。""" + record: dict[str, Any] = { + "event": "stop", + "sid": sid, + "agent": active_agent or "main", + } + if tokens: + record["tokens"] = tokens + if model: + record["model"] = model + if cost_usd is not None: + record["cost_usd"] = cost_usd + if cost_session_usd is not None: + record["cost_session_usd"] = cost_session_usd + if transcript_path: + record["transcript_path"] = transcript_path + if source_offset is not None: + record["source_offset"] = source_offset + if turn_id: + record["turn_id"] = turn_id + if message_id: + record["message_id"] = message_id + if task_slug: + record["task_slug"] = task_slug + if stage: + record["stage"] = stage + return record + + +def build_error_event( + phase: str, + error: str, + sid: str | None = None, +) -> dict[str, Any]: + """构造最小化的 error 事件记录。""" + return { + "event": "error", + "sid": sid or "", + "phase": phase, + "error": error, + } + + +def build_stage_transition_event( + sid: str, + task_slug: str, + stage: str, + status: str | None = None, + executor: str | None = None, + retry_count: int = 0, + review_result: str | None = None, +) -> dict[str, Any]: + """构造 devflow stage 变更事件（只在检测到 devflow 运行、且 stage 真的变化时发出）。 + + 对应 `workflow-state.json` 里某个 stage 的 `status`/`retry_count`/`review_result` + 发生变化——retry_count 上升或 review_result="failed" 是 devflow 里最值得关注的 + 信号（返工/打回循环），比 hook 自身的 `error` 事件更贴近业务语义。 + """ + record: dict[str, Any] = { + "event": "stage_transition", + "sid": sid, + "task_slug": task_slug, + "stage": stage, + } + if status: + record["status"] = status + if executor: + record["executor"] = executor + if retry_count: + record["retry_count"] = int(retry_count) + if review_result: + record["review_result"] = review_result + return record + + +def resolve_overrides_path() -> Path | None: + """返回当前生效的覆盖文件路径：`AOBS_PRICING_OVERRIDES_PATH` 优先，否则默认路径。 + + 返回 `None` 表示"不加载任何覆盖"（纯内置价格表）。支持 `~` 展开。 + """ + raw = os.environ.get("AOBS_PRICING_OVERRIDES_PATH", "").strip() + if raw: + return Path(raw).expanduser() + return DEFAULT_OVERRIDES_PATH + + +def _coerce_price_table(data: Any) -> dict[str, dict[str, float]]: + """把任意 JSON 结果规整成 `{model: {field: float}}`，坏数据按字段/按模型跳过。 + + 与内置表加载的差别只有一处：内置表遇到非数字会整体抛错降级，这里逐字段跳过—— + 用户的补丁文件里写错一个字段不应该把其余正确的覆盖一起丢掉，更不应该让 hook 挂掉。 + """ + if not isinstance(data, dict): + return {} + table: dict[str, dict[str, float]] = {} + for raw_key, raw_value in data.items(): + if not isinstance(raw_value, dict): + continue + key = str(raw_key).strip().lower() + if not key: + continue + row: dict[str, float] = {} + for price_key, price_value in raw_value.items(): + # bool 是 int 的子类，True 会被 float() 变成 1.0，这里按"不是价格"处理。 + if price_value is None or isinstance(price_value, bool): + continue + try: + row[str(price_key).strip().lower()] = float(price_value) + except (TypeError, ValueError): + continue + # 全是坏字段的模型不落表：否则它会被 `lookup_price` 命中成 {}， + # 既算不出成本又不算"未定价"，看板上会变成一个查不到原因的空洞。 + if row: + table[key] = row + return table + + +def load_price_overrides(path: Path | None = None) -> dict[str, dict[str, float]]: + """加载用户定价覆盖文件；任何异常都静默降级为空覆盖（纯内置价格表）。 + + 静默是刻意的：这个文件是给人手写的常驻配置，hook 每次都是新进程， + 一旦因格式问题抛异常/打印，会让整条 hook 链路变得不可用。 + """ + try: + target = path if path is not None else resolve_overrides_path() + if not target: + return {} + with Path(target).expanduser().open("r", encoding="utf-8") as fp: + data = json.load(fp) + except Exception: + return {} + return _coerce_price_table(data) + + +def merge_prices( + base: dict[str, dict[str, float]] | None, + overrides: dict[str, dict[str, float]] | None, +) -> dict[str, dict[str, float]]: + """字段级合并价格表：覆盖里写出的字段替换内置值，没写的字段沿用内置值。 + + 可以新增内置表里不存在的模型；**不支持删除**内置模型或字段（合并只能加不能减）。 + """ + merged: dict[str, dict[str, float]] = {} + for model, row in (base or {}).items(): + if isinstance(row, dict): + merged[model] = dict(row) + for model, patch in (overrides or {}).items(): + if not isinstance(patch, dict): + continue + merged.setdefault(model, {}).update(patch) + return merged + + +def _load_builtin_prices() -> dict[str, dict[str, float]]: + """从 `config/pricing.json` 中一次性加载内置模型价格配置（不含用户覆盖）。""" + path = os.environ.get("AOBS_PRICES_PATH", "").strip() + prices_path = Path(path).expanduser() if path else DEFAULT_PRICES_PATH + try: + with prices_path.open("r", encoding="utf-8") as fp: + data = json.load(fp) + except Exception: + return {} + if not isinstance(data, dict): + return {} + return { + str(key).lower(): {price_key: float(price_value) for price_key, price_value in value.items()} + for key, value in data.items() + if isinstance(value, dict) + } + + +@lru_cache(maxsize=1) +def load_prices() -> dict[str, dict[str, float]]: + """内置价格表 + 用户覆盖文件的合并结果（字段级）。 + + 结果带 `lru_cache`：hook 是短命进程，一次会话里只该读一次盘；长驻进程或单测里 + 改了覆盖文件/环境变量后，调 `clear_price_cache()` 让下一次调用重新加载。 + """ + return merge_prices(_load_builtin_prices(), load_price_overrides()) + + +def clear_price_cache() -> None: + """丢弃 `load_prices()` 的缓存，让下一次调用重读磁盘与环境变量。""" + load_prices.cache_clear() + + +def lookup_price(model: str | None) -> dict[str, float] | None: + """为一个具体模型名找到最匹配的价格配置项。""" + if not model or not isinstance(model, str): + return None + normalized = model.lower().strip() + if not normalized: + return None + prices = load_prices() + if normalized in prices: + return prices[normalized] + best_key: str | None = None + for key in prices: + if key in normalized and (best_key is None or len(key) > len(best_key)): + best_key = key + if best_key is None: + return None + return prices[best_key] + + +def is_unpriced(model: str | None) -> bool: + """模型是否完全没命中价格表（含模糊匹配）。 + + "未定价"的唯一口径就是 `lookup_price(...) is None`（D4）：模糊（最长子串）命中的 + 模型能算出成本，就不该出现在看板的"建议补价"提示里。 + """ + return lookup_price(model) is None + + +def estimate_cost(tokens: dict | None, model: str | None) -> dict[str, Any] | None: + """根据归一化后的 token 总量估算单条 usage 事件的成本。""" + if not isinstance(tokens, dict): + return None + rates = lookup_price(model) + if not rates: + return None + + def _int(key: str) -> int: + try: + return int(tokens.get(key) or 0) + except Exception: + return 0 + + output = _int("output") + cache_read = _int("cache_read") + cache_write = _int("cache_creation") + raw_input = _int("input") + fresh_input = raw_input - cache_read - cache_write if raw_input and raw_input >= (cache_read + cache_write) else raw_input + cost = ( + fresh_input * rates.get("input", 0.0) + + cache_read * rates.get("cache_read", 0.0) + + cache_write * rates.get("cache_write", 0.0) + + output * rates.get("output", 0.0) + ) / 1_000_000.0 + + matched_key = None + normalized = (model or "").lower() + for key in load_prices(): + if key in normalized and (matched_key is None or len(key) > len(matched_key)): + matched_key = key + if matched_key is None and normalized in load_prices(): + matched_key = normalized + + return { + "usd": round(cost, 6), + "model_matched": matched_key, + "rates": rates, + } + + +def cost_of(tokens: dict | None, model: str | None) -> float | None: + """只返回美元成本值的便捷封装。""" + if not tokens or not model: + return None + info = estimate_cost(tokens, model) + if not info: + return None + return info.get("usd") + + +def sum_session_cost(log_path: Path, sid: str) -> float | None: + """尽力根据已发出的事件反算整个 session 的总成本。""" + if not log_path.exists() or not sid: + return None + total = 0.0 + seen = False + try: + with log_path.open("r", encoding="utf-8") as fp: + for line in fp: + if not line.strip().startswith("{"): + continue + try: + rec = json.loads(line) + except Exception: + continue + if rec.get("sid") != sid or rec.get("event") not in {"usage", "tool"}: + continue + cost = rec.get("cost_usd") + if isinstance(cost, (int, float)): + total += float(cost) + seen = True + except Exception: + return None + return round(total, 6) if seen else None diff --git a/.codebuddy/skills/agent-observability/scripts/core/runtime.py b/.codebuddy/skills/agent-observability/scripts/core/runtime.py new file mode 100644 index 0000000..9c647bc --- /dev/null +++ b/.codebuddy/skills/agent-observability/scripts/core/runtime.py @@ -0,0 +1,598 @@ +from __future__ import annotations +"""Hook 主编排入口。 + +这一层负责把外部 hook phase 分发到内部能力： +- session-start / user-prompt-submit / pre / post / stop +- 组装统一运行时路径 +- 串联 collector / emitter / agentlens，并承担 transcript 运行时编排 +""" + +import os +import re +import time +import traceback +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from . import agent_identity, agentlens, collector, devflow, emitter, scanner, state as st + + +@dataclass(frozen=True) +class RuntimePaths: + """运行时常用路径集合，避免每个分支重复计算。""" + base_dir: Path + state_path: Path + pending_path: Path + log_path: Path + + +def base_dir() -> Path: + """返回 hook 运行时使用的根目录。""" + # 运行时产物统一写到 skill 根目录下的 logs/，保持与旧路径兼容。 + return Path(__file__).resolve().parents[2] + + +def build_runtime_paths() -> RuntimePaths: + """构造本次 hook 运行要用到的路径集合。""" + base = base_dir() + return RuntimePaths( + base_dir=base, + state_path=st.get_state_path(base), + pending_path=st.get_pending_path(base), + log_path=emitter.get_log_path(base), + ) + + +def session_id_of(data: dict[str, Any]) -> str: + """兼容不同字段名，提取 session id。""" + return str(data.get("session_id") or data.get("sid") or "") + + +def cwd(data: dict[str, Any]) -> str: + """解析当前项目工作目录。""" + return str(data.get("cwd") or os.environ.get("CODEBUDDY_PROJECT_DIR") or os.getcwd()) + + +def normalize_phase(raw: str | None) -> str: + """把多种 phase 写法折叠成统一内部枚举。""" + phase = (raw or "").strip().lower() + return { + "sessionstart": "session-start", + "session_start": "session-start", + "session-start": "session-start", + "pretooluse": "pre", + "pre": "pre", + "posttooluse": "post", + "post": "post", + "stop": "stop", + "userpromptsubmit": "user-prompt-submit", + "user_prompt_submit": "user-prompt-submit", + "user-prompt-submit": "user-prompt-submit", + "prompt": "user-prompt-submit", + }.get(phase, phase) + + +def emit_error(phase: str, sid: str, err: Exception) -> None: + """把运行时异常写成 error 事件，而不是直接中断 hook。""" + rec = emitter.build_error_event(phase=phase, error=f"{type(err).__name__}: {err}", sid=sid) + rec["x_traceback"] = traceback.format_exc(limit=5)[-1200:] + emitter.emit(build_runtime_paths().log_path, rec) + + +def derive_turn_id(data: dict[str, Any]) -> str: + """优先复用外部已有 id，否则生成一个本地 turn id。""" + for key in ("turn_id", "prompt_id", "request_id", "message_id"): + raw = data.get(key) + if isinstance(raw, str) and raw.strip(): + return raw.strip() + if isinstance(raw, (int, float)) and raw: + return str(raw) + return f"turn-{int(time.time() * 1000)}" + + +def prompt_text_length(data: dict[str, Any]) -> int | None: + """统计用户 prompt 的文本长度，用于 turn 边界指标。""" + for key in ("prompt", "user_prompt", "message", "content", "text"): + val = data.get(key) + if isinstance(val, str): + return len(val) + if isinstance(val, list): + total = 0 + for item in val: + if isinstance(item, dict): + txt = item.get("text") + if isinstance(txt, str): + total += len(txt) + elif isinstance(item, str): + total += len(item) + if total: + return total + return None + + +def extract_prompt_text(data: dict[str, Any]) -> str: + """从 hook payload 中提取可读的 prompt 文本。""" + for key in ("prompt", "user_prompt", "message", "text"): + val = data.get(key) + if isinstance(val, str) and val.strip(): + return val.strip() + if isinstance(val, list): + parts = [] + for item in val: + if isinstance(item, dict): + txt = item.get("text") + if isinstance(txt, str): + parts.append(txt) + elif isinstance(item, str): + parts.append(item) + if parts: + return "\n".join(parts).strip() + return "" + + +def derive_business_scenario(prompt_text: str) -> str | None: + """从 prompt 文本中派生一个简短业务场景标识。""" + if not prompt_text: + return None + + m = re.search(r"标题[：:]\s*(.+?)(?:\s*描述[：:]|\s*$)", prompt_text, re.DOTALL) + if m: + title = m.group(1).strip()[:60] + else: + title = prompt_text[:50].strip() + + sanitized = re.sub(r"[^\w\u4e00-\u9fff\-]", "-", title) + sanitized = re.sub(r"-{2,}", "-", sanitized).strip("-") + return sanitized if sanitized else None + + +def handle_session_start(data: dict[str, Any]) -> None: + """处理 SessionStart：预热 inventory，写 start 事件，初始化 AgentLens。""" + current_sid = session_id_of(data) + if not current_sid: + return + paths = build_runtime_paths() + cwd_value = cwd(data) + skills_meta, rules_meta = scanner.scan_skills_and_rules(cwd_value) + collector.cache_inventory(paths.state_path, current_sid, skills_meta, rules_meta) + emitter.emit(paths.log_path, emitter.build_start_event(sid=current_sid, agent="main")) + agentlens.emit_session_start(state_path=paths.state_path, sid=current_sid, cwd=cwd_value, agent="main") + + +def handle_user_prompt_submit(data: dict[str, Any]) -> None: + """处理 UserPromptSubmit：建立 turn 边界，并打开 AgentLens turn。""" + current_sid = session_id_of(data) + if not current_sid: + return + paths = build_runtime_paths() + cwd_value = cwd(data) + turn_id = derive_turn_id(data) + prompt_len = prompt_text_length(data) + prompt_text = extract_prompt_text(data) + business_scenario = derive_business_scenario(prompt_text) + + emitter.emit( + paths.log_path, + emitter.build_prompt_submit_event(sid=current_sid, turn_id=turn_id, agent="main", prompt_len=prompt_len), + ) + agentlens.emit_turn_start( + state_path=paths.state_path, + sid=current_sid, + cwd=cwd_value, + turn_id=turn_id, + prompt_meta={"prompt_len": prompt_len} if prompt_len is not None else None, + business_scenario=business_scenario, + ) + + +def handle_pre(data: dict[str, Any]) -> None: + """处理 PreToolUse：仅记录 pending，等待 post 配对。""" + collector.record_pre(build_runtime_paths().pending_path, data) + + +def handle_post(data: dict[str, Any]) -> None: + """处理 PostToolUse：计算耗时、归因 agent、发 tool/usage 事件。""" + current_sid = session_id_of(data) + if not current_sid: + return + paths = build_runtime_paths() + cwd_value = cwd(data) + duration_ms = collector.record_post(paths.pending_path, data) + + skills_meta, rules_meta = collector.load_cached_inventory(paths.state_path, current_sid) + if not skills_meta and not rules_meta: + skills_meta, rules_meta = scanner.scan_skills_and_rules(cwd_value) + collector.cache_inventory(paths.state_path, current_sid, skills_meta, rules_meta) + + active_agent, _ = agent_identity.resolve_active_agent_for_event( + state_path=paths.state_path, + sid=current_sid, + cwd=cwd_value, + data=data, + ) + used_skills, used_rules = collector.record_tool_usage( + state_path=paths.state_path, + sid=current_sid, + data=data, + skills_meta=skills_meta, + rules_meta=rules_meta, + active_agent=active_agent, + collect_skills=True, + ) + + tool_name = str(data.get("tool_name") or "") + transcript_path = collector.transcript_path(data) + turn_id = current_turn_id(paths.state_path, current_sid) + event_ts = time.time() + tool_call_id = collector.extract_tool_call_id(data) + dv_ctx = resolve_devflow(paths.state_path, current_sid, cwd_value, paths.log_path) + task_slug = str(dv_ctx.get("task_slug") or "").strip() if dv_ctx else None + stage = str(dv_ctx.get("current_stage") or "").strip() if dv_ctx else None + flush_pending_tool_events( + state_path=paths.state_path, + log_path=paths.log_path, + sid=current_sid, + ) + tool_context = collector.find_current_tool_context( + state_path=paths.state_path, + sid=current_sid, + transcript_path=transcript_path, + tool_name=tool_name, + call_id=tool_call_id, + event_ts=event_ts, + claim=True, + ) + duplicate_tool_context = bool(isinstance(tool_context, dict) and tool_context.get("duplicate")) + if duplicate_tool_context: + tool_context = None + if isinstance(tool_context, dict): + resolved_transcript_path = str(tool_context.get("transcript_path") or "").strip() + if resolved_transcript_path: + transcript_path = resolved_transcript_path + resolved_agent = str(tool_context.get("agent") or "").strip() + if resolved_agent: + active_agent = resolved_agent + + tool_pt_id = str(tool_context.get("pt_id") or "").strip() if isinstance(tool_context, dict) else None + tool_event = emitter.build_tool_event( + sid=current_sid, + tool=tool_name, + duration_ms=duration_ms, + active_agent=active_agent, + transcript_path=transcript_path, + turn_id=turn_id, + task_slug=task_slug, + stage=stage, + pt_id=tool_pt_id, + ) + tool_event["ts"] = event_ts + tool_event["skill"] = used_skills + tool_event["rule"] = used_rules + tool_event["cwd"] = cwd_value + if tool_call_id: + tool_event["call_id"] = tool_call_id + if isinstance(tool_context, dict): + tool_message_id = str(tool_context.get("message_id") or "").strip() or None + if tool_message_id: + tool_event["message_id"] = tool_message_id + tool_details = tool_context.get("tool_details") + if isinstance(tool_details, dict) and tool_details: + tool_event["tool_details"] = dict(tool_details) + + usage_events = emit_transcript_events( + state_path=paths.state_path, + log_path=paths.log_path, + sid=current_sid, + transcript_paths=collector.related_transcript_paths(current_sid, transcript_path), + current_transcript_path=transcript_path, + active_agent=active_agent, + tool_name=tool_name, + event_kind="usage", + max_lines=80, + turn_id=turn_id, + task_slug=task_slug, + stage=stage, + ) + if duplicate_tool_context: + if not usage_events: + return + agentlens.emit_post_step( + state_path=paths.state_path, + sid=current_sid, + tool_event=None, + usage_events=usage_events, + ) + return + if str(tool_event.get("message_id") or "").strip(): + emitter.emit(paths.log_path, tool_event) + agentlens.emit_post_step( + state_path=paths.state_path, + sid=current_sid, + tool_event=tool_event, + usage_events=usage_events, + ) + return + + fallback_usage = collector.find_fallback_usage_event(tool_event, usage_events) + fallback_usage_mid = str((fallback_usage or {}).get("message_id") or "").strip() or None + if fallback_usage_mid: + tool_event["message_id"] = fallback_usage_mid + emitter.emit(paths.log_path, tool_event) + agentlens.emit_post_step( + state_path=paths.state_path, + sid=current_sid, + tool_event=tool_event, + usage_events=usage_events, + ) + return + + st.append_pending_tool_emit(paths.state_path, current_sid, tool_event) + + +def handle_stop(data: dict[str, Any]) -> None: + """处理 Stop：补发 transcript 事件、汇总 session 成本并关闭 AgentLens。""" + current_sid = session_id_of(data) + if not current_sid: + return + paths = build_runtime_paths() + cwd_value = cwd(data) + active_agent, _ = agent_identity.resolve_active_agent_for_event( + state_path=paths.state_path, + sid=current_sid, + cwd=cwd_value, + data=data, + ) + transcript_path = collector.transcript_path(data) + turn_id = current_turn_id(paths.state_path, current_sid) + dv_ctx = resolve_devflow(paths.state_path, current_sid, cwd_value, paths.log_path) + task_slug = str(dv_ctx.get("task_slug") or "").strip() if dv_ctx else None + stage = str(dv_ctx.get("current_stage") or "").strip() if dv_ctx else None + stop_events = emit_transcript_events( + state_path=paths.state_path, + log_path=paths.log_path, + sid=current_sid, + transcript_paths=collector.settled_related_transcript_paths(current_sid, transcript_path), + current_transcript_path=transcript_path, + active_agent=active_agent, + tool_name="__stop__", + event_kind="stop", + max_lines=200, + cost_session=emitter.sum_session_cost(paths.log_path, current_sid), + turn_id=turn_id, + task_slug=task_slug, + stage=stage, + ) + flush_pending_tool_events( + state_path=paths.state_path, + log_path=paths.log_path, + sid=current_sid, + force_emit_fallback=True, + ) + agentlens.emit_session_stop( + state_path=paths.state_path, + sid=current_sid, + cwd=cwd_value, + active_agent=active_agent, + transcript_path=transcript_path, + stop_events=stop_events, + ) + + +def resolve_devflow(state_path: Path, sid: str, cwd_value: str, log_path: Path) -> dict[str, Any] | None: + """解析 devflow 上下文，并把本次观测到的 stage 变更立即发成 `stage_transition` 事件。 + + 任何异常都吞掉、返回 None——devflow 感知是可选增强，绝不能让不跑 devflow 的 + 项目或 workflow-state.json 格式变化导致 hook 报错。 + """ + try: + dv_ctx = devflow.resolve_and_diff(state_path, sid, cwd_value) + except Exception: + return None + if not isinstance(dv_ctx, dict): + return None + task_slug = str(dv_ctx.get("task_slug") or "").strip() + if not task_slug: + return None + for change in dv_ctx.get("changes") or []: + if not isinstance(change, dict): + continue + event = emitter.build_stage_transition_event( + sid=sid, + task_slug=task_slug, + stage=str(change.get("stage") or ""), + status=change.get("status"), + executor=change.get("executor"), + retry_count=int(change.get("retry_count") or 0), + review_result=change.get("review_result"), + ) + emitter.emit(log_path, event) + return dv_ctx + + +def current_turn_id(state_path: Path, sid: str) -> str | None: + """从 state 中读取当前 session 激活中的 turn id。""" + try: + turn = st.get_current_turn(state_path, sid) + except Exception: + turn = None + if isinstance(turn, dict): + raw = turn.get("turn_id") + if isinstance(raw, str) and raw.strip(): + return raw.strip() + return None + + +def flush_pending_tool_events( + *, + state_path: Path, + log_path: Path, + sid: str, + force_emit_fallback: bool = False, +) -> None: + """强制 flush 延迟的 tool 事件，常用于 stop/replay 路径。""" + pending = st.get_pending_tool_emits(state_path, sid) + if not pending: + return + + remaining: list[dict[str, Any]] = [] + for tool_event in pending: + tool_name = str(tool_event.get("tool") or "").strip() + transcript_path = str(tool_event.get("transcript_path") or "").strip() + call_id = str(tool_event.get("call_id") or "").strip() or None + event_ts = tool_event.get("ts") + try: + event_ts_float = float(event_ts) if event_ts is not None else None + except Exception: + event_ts_float = None + + tool_context = collector.find_current_tool_context( + state_path=state_path, + sid=sid, + transcript_path=transcript_path, + tool_name=tool_name, + call_id=call_id, + event_ts=event_ts_float, + claim=True, + ) + if isinstance(tool_context, dict) and tool_context.get("duplicate"): + continue + if isinstance(tool_context, dict): + resolved_transcript_path = str(tool_context.get("transcript_path") or "").strip() + if resolved_transcript_path: + tool_event["transcript_path"] = resolved_transcript_path + resolved_agent = str(tool_context.get("agent") or "").strip() + if resolved_agent: + tool_event["agent"] = resolved_agent + tool_message_id = str(tool_context.get("message_id") or "").strip() or None + if tool_message_id: + tool_event["message_id"] = tool_message_id + tool_details = tool_context.get("tool_details") + if isinstance(tool_details, dict) and tool_details: + tool_event["tool_details"] = dict(tool_details) + emitter.emit(log_path, tool_event) + agentlens.emit_post_step( + state_path=state_path, + sid=sid, + tool_event=tool_event, + usage_events=[], + ) + continue + if force_emit_fallback: + emitter.emit(log_path, tool_event) + agentlens.emit_post_step( + state_path=state_path, + sid=sid, + tool_event=tool_event, + usage_events=[], + ) + continue + remaining.append(tool_event) + + st.replace_pending_tool_emits(state_path, sid, remaining) + + +def emit_transcript_events( + *, + state_path: Path, + log_path: Path, + sid: str, + transcript_paths: list[str], + current_transcript_path: str, + active_agent: str, + tool_name: str, + event_kind: str, + max_lines: int, + cost_session: float | None = None, + turn_id: str | None = None, + task_slug: str | None = None, + stage: str | None = None, +) -> list[dict[str, Any]]: + """发出基于 transcript 增量重建得到的 usage/stop 事件。""" + emitted_events: list[dict[str, Any]] = [] + current_resolved = str(Path(current_transcript_path).resolve()) if current_transcript_path else "" + for path in transcript_paths: + scan = collector.collect_transcript_entries( + state_path=state_path, + sid=sid, + transcript_path=path, + max_lines=max_lines, + ) + entries = scan.get("entries", []) or [] + event_tool = tool_name if event_kind == "usage" and path == current_resolved else "model_request" + event_agent = active_agent if path == current_resolved else agent_identity.agent_for_transcript_path(path, active_agent) + + for idx, entry in enumerate(entries): + event_key = collector.build_transcript_event_key(kind=event_kind, tool=event_tool, entry=entry) + if not st.claim_transcript_event(state_path, sid, event_key, path): + continue + tokens = entry.get("tokens") + model = entry.get("model") + if event_kind == "stop": + event = emitter.build_stop_event( + sid=sid, + tokens=tokens, + model=model, + cost_usd=emitter.cost_of(tokens, model), + cost_session_usd=cost_session if idx == len(entries) - 1 else None, + active_agent=event_agent, + transcript_path=path, + source_offset=int(entry.get("offset", 0) or 0), + turn_id=turn_id, + message_id=str(entry.get("message_id") or "").strip() or None, + task_slug=task_slug, + stage=stage, + ) + else: + event = emitter.build_usage_event( + sid=sid, + tool=event_tool, + tokens=tokens, + active_agent=event_agent, + model=model, + cost_usd=emitter.cost_of(tokens, model), + transcript_path=path, + source_offset=int(entry.get("offset", 0) or 0), + turn_id=turn_id, + message_id=str(entry.get("message_id") or "").strip() or None, + task_slug=task_slug, + stage=stage, + ) + emitter.emit(log_path, event) + emitted_events.append(event) + + st.commit_transcript_progress( + state_path, + sid, + path, + offset=int(scan.get("new_offset", 0) or 0), + last_cumulative_usage=scan.get("last_cumulative"), + ) + return emitted_events + + +def main(argv: list[str]) -> int: + """读取 stdin 输入并按 phase 执行对应 hook 处理分支。""" + phase = normalize_phase(argv[0] if argv else "") + current_sid = "" + try: + data = collector.read_stdin_json() + current_sid = session_id_of(data) + if phase == "session-start": + handle_session_start(data) + elif phase == "user-prompt-submit": + handle_user_prompt_submit(data) + elif phase == "pre": + handle_pre(data) + elif phase == "post": + handle_post(data) + elif phase == "stop": + handle_stop(data) + else: + emitter.emit( + build_runtime_paths().log_path, + emitter.build_error_event(phase=phase or "unknown", error=f"Unsupported phase: {phase!r}", sid=current_sid), + ) + except Exception as exc: + emit_error(phase=phase or "unknown", sid=current_sid, err=exc) + return 0 diff --git a/.codebuddy/skills/agent-observability/scripts/core/scanner.py b/.codebuddy/skills/agent-observability/scripts/core/scanner.py new file mode 100644 index 0000000..86b5305 --- /dev/null +++ b/.codebuddy/skills/agent-observability/scripts/core/scanner.py @@ -0,0 +1,496 @@ +"""Skill / Rule 静态扫描层。 + +这一层只负责扫描项目里的 `.codebuddy/skills` 与 `.codebuddy/rules`： +- 识别有哪些可用 skill / rule +- 提取它们的基础元数据与 frontmatter +- 提供 path-based 命中推断依赖的 inventory 信息 + +它不依赖具体 workflow 语义，本身是通用层。 +""" +from __future__ import annotations + +import os +import re +from pathlib import Path +from typing import Any + +# --- 用于解析 SKILL.md / RULE.mdc frontmatter 的正则 --- +FRONTMATTER_RE = re.compile(r"^---\s*\n(.*?)\n---", re.S) +NAME_FIELD_RE = re.compile(r"^name:\s*(.+)$", re.M) +VERSION_FIELD_RE = re.compile(r"^version:\s*(.+)$", re.M) +AUTHOR_FIELD_RE = re.compile(r"^author:\s*(.+)$", re.M) +TAGS_FIELD_RE = re.compile(r"^tags:\s*\[([^\]]+)\]", re.M) +DESC_FIELD_RE = re.compile(r"^description:\s*(.+?)(?=\n\w+:|\n---|\Z)", re.S | re.M) +ALWAYS_APPLY_RE = re.compile(r"^alwaysApply:\s*(true|false)\s*$", re.M | re.I) +ENABLED_RE = re.compile(r"^enabled:\s*(true|false)\s*$", re.M | re.I) +GLOBS_RE = re.compile(r"^globs:\s*(.+?)(?=\n\w+:|\n---|\Z)", re.S | re.M) +APPLY_AGENTS_RE = re.compile(r"^applyAgents:\s*(.+?)(?=\n\w+:|\n---|\Z)", re.S | re.M) +EXCLUDE_AGENTS_RE = re.compile(r"^excludeAgents:\s*(.+?)(?=\n\w+:|\n---|\Z)", re.S | re.M) + +# 固定的 inventory 根目录（仅限项目内） +PROJECT_SKILLS_REL = Path(".codebuddy/skills") +PROJECT_RULES_REL = Path(".codebuddy/rules") + +# 支持识别的 skill 子模块根目录 +SUBMODULE_DIRS = {"references", "scripts", "templates", "examples", "checklists", "roles"} + +# 允许严格提取的路径类字段键名（不扫描自由文本） +PATH_VALUE_KEYS = { + "file_path", "filePath", "path", "target_file", "target_directory", + "cwd", "rule", "rule_path", "skill_path", +} +PATH_LIST_KEYS = {"paths", "files", "rules", "skills"} + +# 扫描时要跳过的目录 +SKIP_DIR_PARTS = {"skills-by-node", "node_modules", ".git", "__pycache__", "dist"} + +# 可能触发 skill 的工具名 +USE_SKILL_TOOLS = {"use_skill", "UseSkill", "load_skill", "Skill", "skill"} + + +def _is_skipped_path(p: Path) -> bool: + """判断路径是否命中扫描时应跳过的目录集合。""" + parts = set(p.parts) + return bool(parts & SKIP_DIR_PARTS) + + +def _parse_frontmatter(path: Path) -> dict[str, Any]: + """解析 `SKILL.md` 或 `RULE.mdc` 中的 frontmatter。""" + try: + text = path.read_text("utf-8", errors="ignore") + except Exception: + return {} + m = FRONTMATTER_RE.match(text) + if not m: + return {} + block = m.group(1) + info: dict = {} + + def _grab(regex, key, post=None): + mm = regex.search(block) + if mm: + v = mm.group(1).strip().strip('"\'').strip() + info[key] = post(v) if post else v + + _grab(NAME_FIELD_RE, "name") + _grab(VERSION_FIELD_RE, "version") + _grab(AUTHOR_FIELD_RE, "author") + _grab(TAGS_FIELD_RE, "tags", + lambda v: [t.strip().strip('"\'') for t in v.split(",") if t.strip()]) + _grab(DESC_FIELD_RE, "description", lambda v: v.strip().replace("\n", " ")[:200]) + return info + + +def _parse_list_field(block: str, regex: re.Pattern[str]) -> list[str]: + """从 frontmatter 文本块中解析一个逗号分隔的列表字段。""" + mm = regex.search(block) + if not mm: + return [] + raw = mm.group(1).strip().strip('"\'') + if raw.startswith("["): + raw = raw.strip("[]") + return [g.strip().strip('"\'').lower() for g in raw.split(",") if g.strip()] + + +def _parse_rule_frontmatter(rule_path: Path) -> dict[str, Any]: + """解析 `RULE.mdc` 中和规则生效相关的 frontmatter 字段。""" + try: + text = rule_path.read_text("utf-8", errors="ignore") + except Exception: + return {} + m = FRONTMATTER_RE.match(text) + if not m: + return {} + block = m.group(1) + info: dict = {} + + mm = ALWAYS_APPLY_RE.search(block) + if mm: + info["alwaysApply"] = mm.group(1).lower() == "true" + mm = ENABLED_RE.search(block) + if mm: + info["enabled"] = mm.group(1).lower() == "true" + globs = _parse_list_field(block, GLOBS_RE) + if globs: + info["globs"] = globs + + apply_agents = _parse_list_field(block, APPLY_AGENTS_RE) + if apply_agents: + info["applyAgents"] = apply_agents + + exclude_agents = _parse_list_field(block, EXCLUDE_AGENTS_RE) + if exclude_agents: + info["excludeAgents"] = exclude_agents + + mm = DESC_FIELD_RE.search(block) + if mm: + info["description"] = mm.group(1).strip().strip('"\'').replace("\n", " ")[:200] + return info + + +def _classify_source(path: Path, cwd: Path | None) -> str: + """识别来源类型；当前只支持项目内 `.codebuddy` 路径。""" + if not cwd: + return "unknown" + try: + path_str = str(path.resolve()) + cwd_str = str(cwd.resolve()) + except Exception: + return "unknown" + if path_str.startswith(cwd_str + "/.codebuddy/"): + return "project" + return "unknown" + + +def _collect_submodules(skill_dir: Path) -> dict[str, list]: + """收集一个 skill 内部的子模块目录，如 references/scripts/templates。""" + sub: dict = {} + if not skill_dir.is_dir(): + return sub + for kind in ("references", "scripts", "templates", "examples", "checklists", "roles"): + d = skill_dir / kind + if d.is_dir(): + files = [str(f.relative_to(skill_dir)) + for f in d.rglob("*") if f.is_file()] + if files: + sub[kind] = sorted(files)[:30] + return sub + + +def scan_skills_and_rules(cwd: str) -> tuple[dict[str, Any], dict[str, Any]]: + """扫描项目固定根目录：`.codebuddy/skills` 与 `.codebuddy/rules`。""" + skills: dict = {} + rules: dict = {} + cwd_path = Path(cwd) if cwd else None + if not cwd_path or not cwd_path.exists(): + return skills, rules + + skills_root = (cwd_path / PROJECT_SKILLS_REL).resolve() + rules_root = (cwd_path / PROJECT_RULES_REL).resolve() + + if skills_root.is_dir() and not _is_skipped_path(skills_root): + for p in skills_root.rglob("SKILL.md"): + if _is_skipped_path(p): + continue + skill_dir = p.parent + name = skill_dir.name + if not name or name in skills: + continue + meta = _parse_frontmatter(p) + skills[name] = { + "source": _classify_source(skill_dir, cwd_path), + "path": str(skill_dir), + "has_skill_md": True, + "version": meta.get("version"), + "author": meta.get("author"), + "description": meta.get("description"), + "submodules": _collect_submodules(skill_dir), + } + + if rules_root.is_dir() and not _is_skipped_path(rules_root): + for p in rules_root.rglob("*.mdc"): + if _is_skipped_path(p): + continue + name = p.parent.name if p.name == "RULE.mdc" else p.stem + if not name or name in rules: + continue + meta = _parse_rule_frontmatter(p) + rules[name] = { + "source": _classify_source(p.parent, cwd_path), + "path": str(p), + "alwaysApply": meta.get("alwaysApply", False), + "enabled": meta.get("enabled", True), + "globs": meta.get("globs") or [], + "applyAgents": meta.get("applyAgents") or [], + "excludeAgents": meta.get("excludeAgents") or [], + "description": meta.get("description"), + } + + return skills, rules + + +def lookup_skill_meta(skills_meta: dict[str, Any], name: str) -> dict[str, Any] | None: + """从缓存的 inventory 字典里按名称查找 skill 元数据。""" + meta = (skills_meta or {}).get(name) + return meta if isinstance(meta, dict) else None + + +def lookup_rule_meta(rules_meta: dict[str, Any], name: str) -> dict[str, Any] | None: + """从缓存的 inventory 字典里按名称查找 rule 元数据。""" + meta = (rules_meta or {}).get(name) + return meta if isinstance(meta, dict) else None + + +def _collect_path_strings(obj: Any) -> list[str]: + """只收集结构化路径字段，绝不解析自由文本。""" + out: list[str] = [] + + def _walk(x: Any) -> None: + if isinstance(x, dict): + for k, v in x.items(): + if k in PATH_VALUE_KEYS and isinstance(v, str) and v.strip(): + out.append(v.strip()) + elif k in PATH_LIST_KEYS: + if isinstance(v, str) and v.strip(): + out.append(v.strip()) + elif isinstance(v, list): + out.extend([ + s.strip() for s in v + if isinstance(s, str) and s.strip() + ]) + elif isinstance(v, (dict, list)): + _walk(v) + elif isinstance(x, list): + for it in x: + if isinstance(it, (dict, list)): + _walk(it) + + _walk(obj) + return list(dict.fromkeys(out)) + + +def _path_parts(path_str: str) -> list[str]: + """把路径拆成标准化片段列表，便于后续命中判断。""" + p = path_str.replace("\\", "/").strip() + return [part for part in p.split("/") if part] + + +def _extract_skill_from_path(path_str: str) -> str | None: + """从路径中抽取被引用的 skill 名。""" + parts = _path_parts(path_str) + for i in range(len(parts) - 2): + if parts[i] == ".codebuddy" and parts[i + 1] == "skills": + name = parts[i + 2] + if name and re.fullmatch(r"[a-zA-Z0-9_\-]+", name): + return name + return None + + +def _extract_rule_from_path(path_str: str) -> str | None: + """从路径中抽取被引用的 rule 名。""" + parts = _path_parts(path_str) + for i in range(len(parts) - 2): + if parts[i] == ".codebuddy" and parts[i + 1] == "rules": + name = parts[i + 2] + if name.lower().endswith(".mdc"): + name = Path(name).stem + if name and re.fullmatch(r"[a-zA-Z0-9_\-]+", name): + return name + return None + + +def _is_rule_allowed_for_agent(name: str, meta: dict[str, Any], active_agent: str | None) -> bool: + """按角色判断 rule 是否可用；`global` 永远保持可选。""" + if str(name).strip().lower() == "global": + return True + + agent = str(active_agent or "").strip().lower() + if not agent: + return True + + apply_agents = [str(a).strip().lower() for a in (meta.get("applyAgents") or []) if str(a).strip()] + exclude_agents = [str(a).strip().lower() for a in (meta.get("excludeAgents") or []) if str(a).strip()] + + if apply_agents and agent not in apply_agents: + return False + if exclude_agents and agent in exclude_agents: + return False + return True + + +def _glob_matches_any_path(globs: list[str], paths_to_check: list[str]) -> bool: + """判断配置的 glob 是否命中任一收集到的路径。""" + if not globs or not paths_to_check: + return False + import fnmatch + + for p in paths_to_check: + basename = os.path.basename(p) + if any(fnmatch.fnmatch(p, g) or fnmatch.fnmatch(basename, g) for g in globs): + return True + return False + + +def evaluate_rules_for_call( + data: dict, + rules_meta: dict, + active_agent: str | None = None, +) -> dict[str, dict[str, Any]]: + """评估当前工具调用下所有已知 rule，并保留未命中的原因。""" + if not rules_meta: + return {} + + tool_input = data.get("tool_input") or {} + paths_to_check = _collect_path_strings(tool_input) + _, path_rules = extract_paths_from_tool_call(data) + path_rule_set = set(path_rules) + + evaluations: dict[str, dict[str, Any]] = {} + for name, meta in (rules_meta or {}).items(): + if not isinstance(meta, dict): + continue + + globs = [str(g).strip() for g in (meta.get("globs") or []) if str(g).strip()] + apply_agents = [str(a).strip().lower() for a in (meta.get("applyAgents") or []) if str(a).strip()] + exclude_agents = [str(a).strip().lower() for a in (meta.get("excludeAgents") or []) if str(a).strip()] + evaluation: dict[str, Any] = {"matched": False} + + if name in path_rule_set: + evaluation["matched"] = True + evaluation["via"] = "path_inferred" + elif meta.get("enabled") is False: + evaluation["reason"] = "disabled" + elif not _is_rule_allowed_for_agent(name, meta, active_agent): + evaluation["reason"] = "agent_filtered" + if apply_agents: + evaluation["applyAgents"] = apply_agents + if exclude_agents: + evaluation["excludeAgents"] = exclude_agents + elif meta.get("alwaysApply"): + evaluation["matched"] = True + evaluation["via"] = "always_apply" + elif globs: + evaluation["globs"] = globs + evaluation["paths_checked"] = len(paths_to_check) + if not paths_to_check: + evaluation["reason"] = "no_paths" + elif _glob_matches_any_path(globs, paths_to_check): + evaluation["matched"] = True + evaluation["via"] = "glob" + else: + evaluation["reason"] = "glob_not_matched" + evaluation["paths_sample"] = paths_to_check[:3] + else: + evaluation["reason"] = "no_globs" + + evaluations[name] = evaluation + + return evaluations + + +def unmatched_rule_diagnostics_for_call( + data: dict, + rules_meta: dict, + active_agent: str | None = None, +) -> dict[str, dict[str, Any]]: + """返回当前工具调用下未命中的 rule 的精简诊断信息。""" + diagnostics: dict[str, dict[str, Any]] = {} + for name, evaluation in evaluate_rules_for_call(data, rules_meta, active_agent=active_agent).items(): + if evaluation.get("matched"): + continue + item: dict[str, Any] = {"reason": evaluation.get("reason") or "unknown"} + for key in ("globs", "paths_checked", "paths_sample", "applyAgents", "excludeAgents"): + value = evaluation.get(key) + if value is not None and value != []: + item[key] = value + diagnostics[name] = item + return diagnostics + + +def filtered_rules_for_agent(rules_meta: dict, active_agent: str | None = None) -> list[str]: + """列出因 agent 过滤而被排除的 rule（仅看已启用项）。""" + filtered: list[str] = [] + for name, evaluation in evaluate_rules_for_call({}, rules_meta, active_agent=active_agent).items(): + if evaluation.get("reason") == "agent_filtered": + filtered.append(name) + return sorted(set(filtered)) + + +def active_rules_for_call(data: dict, rules_meta: dict, active_agent: str | None = None) -> list[str]: + """判断给定工具调用下哪些 rule 处于生效状态。""" + active: list[str] = [] + for name, evaluation in evaluate_rules_for_call(data, rules_meta, active_agent=active_agent).items(): + if evaluation.get("matched") and evaluation.get("via") != "path_inferred": + active.append(name) + return sorted(set(active)) + + +def extract_skill_from_tool_call(data: dict) -> str | None: + """从类似 use_skill 的工具调用里提取 skill 名称。""" + tool = data.get("tool_name", "") + if tool not in USE_SKILL_TOOLS: + return None + tool_input = data.get("tool_input") or {} + if isinstance(tool_input, dict): + return tool_input.get("command") or tool_input.get("name") or tool_input.get("skill") + return None + + +def is_brainstorming_call(data: dict) -> bool: + """判断当前工具调用是否正在激活或使用 brainstorming skill。""" + direct = extract_skill_from_tool_call(data) + if isinstance(direct, str) and direct.strip().lower() == "brainstorming": + return True + + path_skills, _ = extract_paths_from_tool_call(data) + return any(str(s).strip().lower() == "brainstorming" for s in path_skills) + + +def extract_paths_from_tool_call(data: dict) -> tuple[list[str], list[str]]: + """只根据 `.codebuddy` 根目录下的结构化路径推断 skill/rule。""" + tool_input = data.get("tool_input") or {} + if not isinstance(tool_input, dict): + return [], [] + + skills: list[str] = [] + rules: list[str] = [] + for p in _collect_path_strings(tool_input): + s = _extract_skill_from_path(p) + if s: + skills.append(s) + r = _extract_rule_from_path(p) + if r: + rules.append(r) + + return list(dict.fromkeys(skills)), list(dict.fromkeys(rules)) + + +def extract_submodule_hits(data: dict) -> list[dict[str, str]]: + """仅从结构化 skill 路径中提取子模块命中结果，不扫描自由文本。""" + tool_input = data.get("tool_input") or {} + if not isinstance(tool_input, dict): + return [] + + seen: set[tuple[str, str]] = set() + hits: list[dict[str, str]] = [] + path_values = _collect_path_strings(tool_input) + + for p in path_values: + parts = _path_parts(p) + for i in range(len(parts) - 3): + if parts[i] == ".codebuddy" and parts[i + 1] == "skills": + skill_name = parts[i + 2] + sub_root = parts[i + 3] + if not re.fullmatch(r"[a-zA-Z0-9_\-]+", skill_name): + continue + if sub_root not in SUBMODULE_DIRS: + continue + sub_path = "/".join(parts[i + 3:]).rstrip("/") + key = (skill_name, sub_path) + if key in seen: + continue + seen.add(key) + hits.append({"skill": skill_name, "submodule": sub_path}) + break + + return hits + + +def extract_bash_skill_scripts(data: dict) -> list[str]: + """从命令文本中的显式 `.codebuddy/skills` 路径识别 skill 脚本。""" + if data.get("tool_name") not in ("Bash", "bash", "execute_command"): + return [] + tool_input = data.get("tool_input") or {} + if not isinstance(tool_input, dict): + return [] + cmd = tool_input.get("command") or "" + if not isinstance(cmd, str) or not cmd.strip(): + return [] + + skills: list[str] = [] + for token in cmd.split(): + t = token.strip("\"'`;,()[]{}") + s = _extract_skill_from_path(t) + if s: + skills.append(s) + return list(dict.fromkeys(skills)) diff --git a/.codebuddy/skills/agent-observability/scripts/core/state.py b/.codebuddy/skills/agent-observability/scripts/core/state.py new file mode 100644 index 0000000..5e7d33e --- /dev/null +++ b/.codebuddy/skills/agent-observability/scripts/core/state.py @@ -0,0 +1,1222 @@ +"""持久化状态层。 + +这一层负责跨 hook 调用保存与协调状态： +- session 级 skill/rule 使用统计 +- 当前 agent、派发关系与历史 +- transcript offset、去重 claim、pending emits +- AgentLens sidecar 状态 +""" +from __future__ import annotations + +import json +import time +from contextlib import contextmanager +from pathlib import Path +from typing import Any + +try: + import fcntl +except Exception: # pragma: no cover - non-posix fallback + fcntl = None # type: ignore + + +def get_state_path(base_dir: Path) -> Path: + """返回主状态文件路径。""" + log_dir = base_dir / "logs" + log_dir.mkdir(parents=True, exist_ok=True) + return log_dir / ".state.json" + + +def get_pending_path(base_dir: Path) -> Path: + """返回 Pre/PostToolUse 配对使用的 pending 文件路径。""" + log_dir = base_dir / "logs" + log_dir.mkdir(parents=True, exist_ok=True) + return log_dir / ".pending_calls.json" + + +def load_state(state_path: Path) -> dict[str, Any]: + """从磁盘加载状态；任何异常都回退为空字典。""" + if state_path.exists(): + try: + return json.loads(state_path.read_text("utf-8")) + except Exception: + return {} + return {} + + +def save_state(state_path: Path, state: dict[str, Any]) -> None: + """把状态写回磁盘。""" + state_path.write_text(json.dumps(state, ensure_ascii=False), encoding="utf-8") + + +def get_state_lock_path(state_path: Path) -> Path: + """返回状态写锁文件路径。""" + return state_path.parent / ".state.lock" + + +def get_transcript_seen_path(state_path: Path) -> Path: + """返回 transcript usage 事件 claim 的 sidecar 台账文件。 + + 这部分单独存放，不混进 `.state.json`，是为了避免并发 hook 写状态时， + 旧 state 覆盖掉已经 claim 过的 transcript offset。 + """ + return state_path.parent / ".transcript_events_seen.json" + + +@contextmanager +def state_lock(state_path: Path): + """为状态修改提供跨进程建议锁（独占）。""" + lock_path = get_state_lock_path(state_path) + lock_path.parent.mkdir(parents=True, exist_ok=True) + with lock_path.open("a+", encoding="utf-8") as fp: + if fcntl is not None: + fcntl.flock(fp.fileno(), fcntl.LOCK_EX) + try: + yield + finally: + if fcntl is not None: + fcntl.flock(fp.fileno(), fcntl.LOCK_UN) + + +@contextmanager +def state_read_lock(state_path: Path): + """为状态一致性读取提供跨进程共享读锁。""" + lock_path = get_state_lock_path(state_path) + lock_path.parent.mkdir(parents=True, exist_ok=True) + with lock_path.open("a+", encoding="utf-8") as fp: + if fcntl is not None: + fcntl.flock(fp.fileno(), fcntl.LOCK_SH) + try: + yield + finally: + if fcntl is not None: + fcntl.flock(fp.fileno(), fcntl.LOCK_UN) + + +def update_state_locked(state_path: Path, updater): + """在跨进程锁保护下完成 load -> mutate -> save。""" + with state_lock(state_path): + state = load_state(state_path) + result = updater(state) + save_state(state_path, state) + return result + + +def load_pending(pending_path: Path) -> dict[str, Any]: + """加载待配对的 pre/post 数据。""" + if pending_path.exists(): + try: + return json.loads(pending_path.read_text("utf-8")) + except Exception: + return {} + return {} + + +def save_pending(pending_path: Path, data: dict[str, Any]) -> None: + """保存待配对的 pre/post 数据。""" + pending_path.write_text(json.dumps(data), encoding="utf-8") + + +def ensure_session(state: dict, sid: str) -> dict: + """确保某个 session 的状态结构存在，并返回该 session 字典。 + + Session 结构： + { + "skills": {name: {count, first_ts, last_ts, tools, via, source, by_agent}}, + "rules": {name: {count, first_ts, last_ts, tools, via, source, by_agent}}, + "started_at": float, + "current_agent": str, + "agent_history": [{ts, agent, evidence}], + "dispatched": {agent_name: count}, + "_skills_meta": {...}, # SessionStart 时缓存的 inventory + "_rules_meta": {...}, + } + """ + if sid not in state: + state[sid] = { + "skills": {}, + "rules": {}, + "started_at": time.time(), + "current_agent": "main", + "agent_history": [], + "dispatched": {}, + } + sess = state[sid] + # 保证字段类型正确，兼容旧数据 + if not isinstance(sess.get("skills"), dict): + sess["skills"] = {} + if not isinstance(sess.get("rules"), dict): + sess["rules"] = {} + sess.setdefault("current_agent", "main") + sess.setdefault("agent_history", []) + sess.setdefault("dispatched", {}) + if not isinstance(sess.get("_pending_tool_emits"), list): + sess["_pending_tool_emits"] = [] + return sess + + +def get_devflow_state(state: dict[str, Any], sid: str) -> dict[str, Any]: + """返回某个 session 缓存的 devflow 上下文与最近一次 stage 快照（见 devflow.py）。""" + sess = state.get(sid) + if not isinstance(sess, dict): + return {} + dv = sess.get("_devflow") + return dv if isinstance(dv, dict) else {} + + +def set_devflow_state(state: dict[str, Any], sid: str, updates: dict[str, Any]) -> None: + """合并写入某个 session 的 devflow 缓存字段（`checked` / `context` / `last_snapshot`）。""" + sess = ensure_session(state, sid) + dv = sess.get("_devflow") + if not isinstance(dv, dict): + dv = {} + sess["_devflow"] = dv + dv.update(updates) + + +def get_agentlens_session(state: dict[str, Any], sid: str) -> dict[str, Any]: + """返回某个 session 的 AgentLens sidecar payload。""" + sess = ensure_session(state, sid) + payload = sess.setdefault("_agentlens", {}) + if not isinstance(payload, dict): + payload = {} + sess["_agentlens"] = payload + return payload + + +AGENTLENS_SCHEMA_VERSION = 2 +_TURN_HISTORY_MAX = 5 + + +def _migrate_agentlens_v1_to_v2(payload: dict[str, Any], sess: dict[str, Any] | None) -> dict[str, Any]: + """把 v1 版 ``_agentlens`` payload 转成以 turn 为中心的 v2 布局。 + + v1 布局（pre-turn）： + {"carrier": {...}, "enabled": bool, "session_id": "...", "app_name": "..."} + v2 布局（turn-centric）： + { + "version": 2, + "session_id": "...", + "enabled": bool, + "app_name": "...", + "current_turn": {"turn_id": "legacy", "started_at": ..., "carrier": {...}, "subagent_spans": {}, "agent_spans": {}, "step_spans": {}}, + "turn_history": [], + } + + 幂等约束：如果已经带有 ``version``，就原样返回。 + """ + if not isinstance(payload, dict): + return {"version": AGENTLENS_SCHEMA_VERSION, "enabled": False, "current_turn": None, "turn_history": []} + if payload.get("version") == AGENTLENS_SCHEMA_VERSION: + payload.setdefault("current_turn", None) + payload.setdefault("turn_history", []) + current = payload.get("current_turn") + if isinstance(current, dict): + current.setdefault("subagent_spans", {}) + current.setdefault("agent_spans", {}) + current.setdefault("step_spans", {}) + return payload + + migrated: dict[str, Any] = { + "version": AGENTLENS_SCHEMA_VERSION, + "session_id": payload.get("session_id"), + "enabled": bool(payload.get("enabled", False)), + } + if "app_name" in payload: + migrated["app_name"] = payload.get("app_name") + if "last_error" in payload: + migrated["last_error"] = payload.get("last_error") + + legacy_carrier = payload.get("carrier") + if isinstance(legacy_carrier, dict) and legacy_carrier: + started_at = None + if isinstance(sess, dict): + try: + started_at = float(sess.get("started_at") or 0) or None + except Exception: + started_at = None + migrated["current_turn"] = { + "turn_id": "legacy", + "started_at": started_at or time.time(), + "carrier": dict(legacy_carrier), + "subagent_spans": {}, + "agent_spans": {}, + "step_spans": {}, + } + else: + migrated["current_turn"] = None + migrated["turn_history"] = [] + return migrated + + +def load_agentlens_session(state_path: Path, sid: str) -> dict[str, Any]: + """加载某个 session 已持久化的 AgentLens sidecar payload。 + + 这里使用共享读锁，避免并发 ``update_agentlens_session`` 写入时读到撕裂数据。 + 返回值会透明地把 v1 payload 迁移成 v2 副本；真正落盘迁移会在下一次写入时完成。 + """ + with state_read_lock(state_path): + state = load_state(state_path) + sess = state.get(sid) + if not isinstance(sess, dict): + return {} + payload = sess.get("_agentlens") + if not isinstance(payload, dict): + return {} + # 先给调用方返回迁移后的副本，真正持久化迁移留到后续写入时完成。 + return _migrate_agentlens_v1_to_v2(dict(payload), sess) + + +def update_agentlens_session( + state_path: Path, + sid: str, + updates: dict[str, Any], + *, + clear_keys: list[str] | None = None, +) -> dict[str, Any]: + """原子地持久化某个 session 的 AgentLens sidecar 字段。 + + 在合并 updates 之前会先做 v1→v2 迁移，这样磁盘上的 payload 会随着写入自然收敛到 v2。 + """ + + def _update(state: dict[str, Any]) -> dict[str, Any]: + sess = ensure_session(state, sid) + payload = get_agentlens_session(state, sid) + # 如果仍是 v1，就原地迁移。 + if payload.get("version") != AGENTLENS_SCHEMA_VERSION: + migrated = _migrate_agentlens_v1_to_v2(dict(payload), sess) + payload.clear() + payload.update(migrated) + for key in clear_keys or []: + payload.pop(key, None) + for key, value in updates.items(): + if value is None: + payload.pop(key, None) + else: + payload[key] = value + payload["version"] = AGENTLENS_SCHEMA_VERSION + return dict(payload) + + return update_state_locked(state_path, _update) + + +def begin_turn( + state_path: Path, + sid: str, + turn_id: str, + carrier: dict[str, str], + *, + started_at: float | None = None, + max_history: int = _TURN_HISTORY_MAX, +) -> dict[str, Any]: + """在独占锁下打开一个新的 turn。 + + 语义： + - 如果存在 ``current_turn``，先把它归档进有上限的 ``turn_history``。 + - 为新 turn 重置 ``subagent_spans`` / ``agent_spans`` / ``step_spans``。 + - 对相同 ``turn_id`` 保持幂等：重复调用直接返回已有 ``current_turn``， + 不重新生成 trace_id。 + + 返回最终生成的 ``current_turn`` 字典。 + """ + ts = float(started_at if started_at is not None else time.time()) + + def _update(state: dict[str, Any]) -> dict[str, Any]: + sess = ensure_session(state, sid) + payload = get_agentlens_session(state, sid) + if payload.get("version") != AGENTLENS_SCHEMA_VERSION: + migrated = _migrate_agentlens_v1_to_v2(dict(payload), sess) + payload.clear() + payload.update(migrated) + + current = payload.get("current_turn") + # 幂等：如果同一个 turn_id 还在处理中，就原样返回。 + if isinstance(current, dict) and str(current.get("turn_id") or "") == str(turn_id): + return dict(current) + + # 归档上一个 turn。 + if isinstance(current, dict) and current.get("turn_id"): + history = payload.setdefault("turn_history", []) + if not isinstance(history, list): + history = [] + payload["turn_history"] = history + archived = { + "turn_id": current.get("turn_id"), + "started_at": current.get("started_at"), + "ended_at": ts, + "carrier_traceparent": (current.get("carrier") or {}).get("traceparent"), + } + history.append(archived) + if len(history) > max_history: + del history[: len(history) - max_history] + + new_turn = { + "turn_id": str(turn_id), + "started_at": ts, + "carrier": dict(carrier or {}), + "subagent_spans": {}, + "agent_spans": {}, + "step_spans": {}, + } + payload["current_turn"] = new_turn + payload["version"] = AGENTLENS_SCHEMA_VERSION + return dict(new_turn) + + return update_state_locked(state_path, _update) + + +def upsert_subagent_span( + state_path: Path, + sid: str, + role: str, + carrier: dict[str, str], +) -> dict[str, str]: + """在当前 turn 下登记 subagent 的 ``invoke_agent`` 子 span carrier。 + + 对 `(turn, role)` 维度保持幂等：如果活跃 turn 下已经记录了该角色的 carrier， + 就直接返回已有值，不再重复创建。 + """ + now = time.time() + + def _update(state: dict[str, Any]) -> dict[str, str]: + payload = get_agentlens_session(state, sid) + if payload.get("version") != AGENTLENS_SCHEMA_VERSION: + migrated = _migrate_agentlens_v1_to_v2(dict(payload), state.get(sid) if isinstance(state.get(sid), dict) else None) + payload.clear() + payload.update(migrated) + current = payload.get("current_turn") + if not isinstance(current, dict): + # 没有活跃 turn，无法登记 subagent span。 + return {} + subs = current.setdefault("subagent_spans", {}) + if not isinstance(subs, dict): + subs = {} + current["subagent_spans"] = subs + existing = subs.get(role) + if isinstance(existing, dict) and isinstance(existing.get("carrier"), dict) and existing["carrier"]: + return dict(existing["carrier"]) + subs[role] = {"carrier": dict(carrier or {}), "opened_at": now} + payload["version"] = AGENTLENS_SCHEMA_VERSION + return dict(carrier or {}) + + return update_state_locked(state_path, _update) + + +def upsert_subagent_span_atomic( + state_path: Path, + sid: str, + role: str, + *, + carrier_factory: Any, +) -> dict[str, str]: + """在写锁内原子地获取或创建 subagent 的 ``invoke_agent`` carrier。 + + `carrier_factory` 是一个无参可调用对象，用来生成 carrier 字典。 + 只有在 span 还不存在时才会被调用，从而避免这样的 TOCTOU 竞争： + ``_generate_subagent_carrier`` 先产生了不可逆的 zhiyanllm ``invoke_agent`` span， + 随后另一个 hook 进程已经写入同一角色，导致本次创建变成孤儿。 + """ + now = time.time() + + def _update(state: dict[str, Any]) -> dict[str, str]: + payload = get_agentlens_session(state, sid) + if payload.get("version") != AGENTLENS_SCHEMA_VERSION: + migrated = _migrate_agentlens_v1_to_v2(dict(payload), state.get(sid) if isinstance(state.get(sid), dict) else None) + payload.clear() + payload.update(migrated) + current = payload.get("current_turn") + if not isinstance(current, dict): + return {} + subs = current.setdefault("subagent_spans", {}) + if not isinstance(subs, dict): + subs = {} + current["subagent_spans"] = subs + existing = subs.get(role) + if isinstance(existing, dict) and isinstance(existing.get("carrier"), dict) and existing["carrier"]: + return dict(existing["carrier"]) + new_carrier = carrier_factory() + if not isinstance(new_carrier, dict) or not new_carrier.get("traceparent"): + return {} + subs[role] = {"carrier": dict(new_carrier), "opened_at": now} + payload["version"] = AGENTLENS_SCHEMA_VERSION + return dict(new_carrier) + + return update_state_locked(state_path, _update) + + +def get_current_turn(state_path: Path, sid: str) -> dict[str, Any] | None: + """返回当前活跃的 turn 字典；如果没有打开 turn，就返回 None。""" + payload = load_agentlens_session(state_path, sid) + current = payload.get("current_turn") + return current if isinstance(current, dict) else None + + +def get_subagent_span_carrier(state_path: Path, sid: str, role: str) -> dict[str, str] | None: + """返回活跃 turn 下已持久化的 subagent span carrier；如果没有则返回 None。""" + turn = get_current_turn(state_path, sid) + if not isinstance(turn, dict): + return None + subs = turn.get("subagent_spans") + if not isinstance(subs, dict): + return None + rec = subs.get(role) + if not isinstance(rec, dict): + return None + carrier = rec.get("carrier") + return dict(carrier) if isinstance(carrier, dict) and carrier else None + + +def _empty_agent_aggregate() -> dict[str, Any]: + """创建一份空的 agent 聚合指标骨架。""" + return { + "tokens": { + "input": 0, + "output": 0, + "cache_read": 0, + "cache_creation": 0, + "total": 0, + }, + "event_count": 0, + "cost_usd": 0.0, + "tool_duration_ms": 0, + "llm_call_count": 0, + "tool_call_count": 0, + "step_count": 0, + } + + +def _normalize_agent(agent: str | None) -> str: + """把 agent 名归一化，空值统一落到 ``main``。""" + return str(agent or "main").strip() or "main" + + +def _ensure_turn_agent_spans(current: dict[str, Any]) -> dict[str, Any]: + """确保当前 turn 下存在 `agent_spans` 容器。""" + agent_spans = current.setdefault("agent_spans", {}) + if not isinstance(agent_spans, dict): + agent_spans = {} + current["agent_spans"] = agent_spans + return agent_spans + + +def _ensure_agent_span_rec(current: dict[str, Any], agent: str) -> dict[str, Any]: + """确保指定 agent 在当前 turn 下有一条完整聚合记录。""" + agent_spans = _ensure_turn_agent_spans(current) + rec = agent_spans.get(agent) + if not isinstance(rec, dict): + rec = {} + agent_spans[agent] = rec + carrier = rec.get("carrier") + if not isinstance(carrier, dict): + rec["carrier"] = {} + aggregate = rec.get("aggregate") + if not isinstance(aggregate, dict): + rec["aggregate"] = _empty_agent_aggregate() + return rec + + +def get_agent_span_carrier(state_path: Path, sid: str, agent: str) -> dict[str, str] | None: + """返回活跃 turn 下已持久化的 agent 分组 span carrier；如果没有则返回 None。""" + turn = get_current_turn(state_path, sid) + if not isinstance(turn, dict): + return None + agent_spans = turn.get("agent_spans") + if not isinstance(agent_spans, dict): + return None + rec = agent_spans.get(_normalize_agent(agent)) + if not isinstance(rec, dict): + return None + carrier = rec.get("carrier") + return dict(carrier) if isinstance(carrier, dict) and carrier else None + + +def upsert_agent_span_atomic( + state_path: Path, + sid: str, + agent: str, + *, + carrier_factory: Any, +) -> dict[str, str]: + """原子地获取或创建 agent 分组 span carrier。""" + now = time.time() + normalized_agent = _normalize_agent(agent) + + def _update(state: dict[str, Any]) -> dict[str, str]: + payload = get_agentlens_session(state, sid) + if payload.get("version") != AGENTLENS_SCHEMA_VERSION: + migrated = _migrate_agentlens_v1_to_v2( + dict(payload), + state.get(sid) if isinstance(state.get(sid), dict) else None, + ) + payload.clear() + payload.update(migrated) + current = payload.get("current_turn") + if not isinstance(current, dict): + return {} + rec = _ensure_agent_span_rec(current, normalized_agent) + existing = rec.get("carrier") + if isinstance(existing, dict) and existing.get("traceparent"): + return dict(existing) + new_carrier = carrier_factory() + if not isinstance(new_carrier, dict) or not new_carrier.get("traceparent"): + return {} + rec["carrier"] = dict(new_carrier) + rec.setdefault("opened_at", now) + payload["version"] = AGENTLENS_SCHEMA_VERSION + return dict(new_carrier) + + return update_state_locked(state_path, _update) + + +def bump_agent_aggregate( + state_path: Path, + sid: str, + agent: str, + *, + usage_event: dict[str, Any] | None = None, + tool_event: dict[str, Any] | None = None, + step_created: bool = False, +) -> dict[str, Any]: + """增加活跃 turn 下按 agent 聚合的统计指标。""" + normalized_agent = _normalize_agent(agent) + + def _int(value: Any) -> int: + try: + return int(value or 0) + except Exception: + return 0 + + def _float(value: Any) -> float: + try: + return float(value or 0) + except Exception: + return 0.0 + + def _update(state: dict[str, Any]) -> dict[str, Any]: + payload = get_agentlens_session(state, sid) + if payload.get("version") != AGENTLENS_SCHEMA_VERSION: + migrated = _migrate_agentlens_v1_to_v2( + dict(payload), + state.get(sid) if isinstance(state.get(sid), dict) else None, + ) + payload.clear() + payload.update(migrated) + current = payload.get("current_turn") + if not isinstance(current, dict): + return {} + rec = _ensure_agent_span_rec(current, normalized_agent) + aggregate = rec.get("aggregate") + if not isinstance(aggregate, dict): + aggregate = _empty_agent_aggregate() + rec["aggregate"] = aggregate + tokens = aggregate.setdefault("tokens", {}) + if not isinstance(tokens, dict): + tokens = {} + aggregate["tokens"] = tokens + + if step_created: + aggregate["step_count"] = _int(aggregate.get("step_count")) + 1 + + if isinstance(tool_event, dict): + aggregate["tool_call_count"] = _int(aggregate.get("tool_call_count")) + 1 + ms = tool_event.get("ms") + if ms is not None: + aggregate["tool_duration_ms"] = _int(aggregate.get("tool_duration_ms")) + _int(ms) + + if isinstance(usage_event, dict): + raw_tokens = usage_event.get("tokens") + if isinstance(raw_tokens, dict): + # 聚合 agent 摘要里需要暴露的字段。 + for key in ("input", "output", "cache_read", "cache_creation"): + tokens[key] = _int(tokens.get(key)) + _int(raw_tokens.get(key)) + # total = input + output，其中 input 已经包含 cache_read。 + tokens["total"] = _int(tokens.get("input")) + _int(tokens.get("output")) + aggregate["llm_call_count"] = _int(aggregate.get("llm_call_count")) + 1 + aggregate["event_count"] = _int(aggregate.get("event_count")) + 1 + aggregate["cost_usd"] = round(_float(aggregate.get("cost_usd")) + _float(usage_event.get("cost_usd")), 10) + + payload["version"] = AGENTLENS_SCHEMA_VERSION + return json.loads(json.dumps(aggregate, ensure_ascii=False)) + + return update_state_locked(state_path, _update) + + +def get_subagent_aggregate(state_path: Path, sid: str, role: str) -> dict[str, Any] | None: + """返回活跃 turn 下当前 agent 的聚合摘要。""" + turn = get_current_turn(state_path, sid) + if not isinstance(turn, dict): + return None + agent_spans = turn.get("agent_spans") + if not isinstance(agent_spans, dict): + return None + rec = agent_spans.get(_normalize_agent(role)) + if not isinstance(rec, dict): + return None + aggregate = rec.get("aggregate") + if not isinstance(aggregate, dict): + return None + return json.loads(json.dumps(aggregate, ensure_ascii=False)) + + +def _step_span_key(agent: str, message_id: str) -> str: + """为 step span 生成按 agent 与 message_id 唯一定位的 key。""" + return f"{agent}|{message_id}" + + +def upsert_step_span( + state_path: Path, + sid: str, + agent: str, + message_id: str, + carrier: dict[str, str], + *, + transcript_path: str | None = None, +) -> dict[str, str]: + """在当前 turn 下登记一个 step span carrier，并对 `(turn, agent, message_id)` 保持幂等。""" + now = time.time() + normalized_agent = str(agent or "main").strip() or "main" + normalized_message_id = str(message_id or "").strip() + if not normalized_message_id: + return {} + + def _update(state: dict[str, Any]) -> dict[str, str]: + payload = get_agentlens_session(state, sid) + if payload.get("version") != AGENTLENS_SCHEMA_VERSION: + migrated = _migrate_agentlens_v1_to_v2(dict(payload), state.get(sid) if isinstance(state.get(sid), dict) else None) + payload.clear() + payload.update(migrated) + current = payload.get("current_turn") + if not isinstance(current, dict): + return {} + steps = current.setdefault("step_spans", {}) + if not isinstance(steps, dict): + steps = {} + current["step_spans"] = steps + key = _step_span_key(normalized_agent, normalized_message_id) + existing = steps.get(key) + if isinstance(existing, dict) and isinstance(existing.get("carrier"), dict) and existing["carrier"]: + return dict(existing["carrier"]) + steps[key] = { + "carrier": dict(carrier or {}), + "opened_at": now, + "agent": normalized_agent, + "message_id": normalized_message_id, + "transcript_path": transcript_path or "", + } + payload["version"] = AGENTLENS_SCHEMA_VERSION + return dict(carrier or {}) + + return update_state_locked(state_path, _update) + + +def upsert_step_span_atomic( + state_path: Path, + sid: str, + agent: str, + message_id: str, + *, + carrier_factory: Any, + transcript_path: str | None = None, +) -> dict[str, str]: + """在写锁内原子地获取或创建一个 step span carrier。 + + `carrier_factory` 是一个无参可调用对象，用来生成 carrier 字典。 + 只有在 span 还不存在时才会被调用，从而避免这样的 TOCTOU 竞争： + ``_generate_step_carrier`` 已经先产生了不可逆的 zhiyanllm span， + 但随后另一个 hook 进程已经写入同一 step，导致本次创建失去归属。 + """ + now = time.time() + normalized_agent = str(agent or "main").strip() or "main" + normalized_message_id = str(message_id or "").strip() + if not normalized_message_id: + return {} + + def _update(state: dict[str, Any]) -> dict[str, str]: + payload = get_agentlens_session(state, sid) + if payload.get("version") != AGENTLENS_SCHEMA_VERSION: + migrated = _migrate_agentlens_v1_to_v2(dict(payload), state.get(sid) if isinstance(state.get(sid), dict) else None) + payload.clear() + payload.update(migrated) + current = payload.get("current_turn") + if not isinstance(current, dict): + return {} + steps = current.setdefault("step_spans", {}) + if not isinstance(steps, dict): + steps = {} + current["step_spans"] = steps + key = _step_span_key(normalized_agent, normalized_message_id) + existing = steps.get(key) + if isinstance(existing, dict) and isinstance(existing.get("carrier"), dict) and existing["carrier"]: + return dict(existing["carrier"]) + new_carrier = carrier_factory() + if not isinstance(new_carrier, dict) or not new_carrier.get("traceparent"): + return {} + steps[key] = { + "carrier": dict(new_carrier), + "opened_at": now, + "agent": normalized_agent, + "message_id": normalized_message_id, + "transcript_path": transcript_path or "", + } + payload["version"] = AGENTLENS_SCHEMA_VERSION + return dict(new_carrier) + + return update_state_locked(state_path, _update) + + +def get_step_span_carrier(state_path: Path, sid: str, agent: str, message_id: str) -> dict[str, str] | None: + """返回活跃 turn 下已持久化的 step span carrier；如果没有则返回 None。""" + turn = get_current_turn(state_path, sid) + if not isinstance(turn, dict): + return None + steps = turn.get("step_spans") + if not isinstance(steps, dict): + return None + key = _step_span_key(str(agent or "main").strip() or "main", str(message_id or "").strip()) + rec = steps.get(key) + if not isinstance(rec, dict): + return None + carrier = rec.get("carrier") + return dict(carrier) if isinstance(carrier, dict) and carrier else None + + +def get_latest_session_id(state: dict[str, Any]) -> str | None: + """返回 state 中最近启动的 session id。""" + latest_sid: str | None = None + latest_started_at = -1.0 + for sid, sess in state.items(): + if sid.startswith("_") or not isinstance(sess, dict): + continue + started_at = float(sess.get("started_at", 0) or 0) + if started_at >= latest_started_at: + latest_started_at = started_at + latest_sid = sid + return latest_sid + + +def get_pending_tool_emits(state_path: Path, sid: str) -> list[dict[str, Any]]: + """返回等待与 transcript/message_id 关联的缓冲 tool 事件。""" + with state_read_lock(state_path): + state = load_state(state_path) + sess = state.get(sid) + if not isinstance(sess, dict): + return [] + pending = sess.get("_pending_tool_emits") + if not isinstance(pending, list): + return [] + return [dict(item) for item in pending if isinstance(item, dict)] + + +def append_pending_tool_emit(state_path: Path, sid: str, tool_event: dict[str, Any]) -> list[dict[str, Any]]: + """缓存一条 tool 事件，延后再做 message_id 关联。""" + + def _update(state: dict[str, Any]) -> list[dict[str, Any]]: + sess = ensure_session(state, sid) + pending = sess.setdefault("_pending_tool_emits", []) + if not isinstance(pending, list): + pending = [] + sess["_pending_tool_emits"] = pending + pending.append(dict(tool_event)) + return [dict(item) for item in pending if isinstance(item, dict)] + + return update_state_locked(state_path, _update) + + +def replace_pending_tool_emits(state_path: Path, sid: str, tool_events: list[dict[str, Any]]) -> list[dict[str, Any]]: + """覆盖某个 session 的缓冲 tool 事件队列。""" + + def _update(state: dict[str, Any]) -> list[dict[str, Any]]: + sess = ensure_session(state, sid) + normalized = [dict(item) for item in tool_events if isinstance(item, dict)] + sess["_pending_tool_emits"] = normalized + return [dict(item) for item in normalized] + + return update_state_locked(state_path, _update) + +def bump_skill( + sess: dict, + name: str, + *, + via: str, + tool: str | None, + meta: dict | None = None, + submodule: str | None = None, + agent: str | None = None, +) -> None: + """增加 skill 使用计数，并按需记录子模块与 agent 维度。""" + if not name: + return + now = time.time() + rec = sess["skills"].setdefault(name, { + "count": 0, + "first_ts": now, + "last_ts": now, + "tools": [], + "via": [], + "source": None, + "version": None, + "submodule_hits": {}, + "by_agent": {}, + }) + rec["count"] += 1 + rec["last_ts"] = now + if tool and tool not in rec["tools"]: + rec["tools"].append(tool) + if via not in rec.get("via", []): + rec.setdefault("via", []).append(via) + if meta: + if meta.get("source") and not rec.get("source"): + rec["source"] = meta["source"] + if meta.get("version") and not rec.get("version"): + rec["version"] = meta["version"] + # 子模块命中统计 + if submodule: + sub_dict = rec.setdefault("submodule_hits", {}) + if not isinstance(sub_dict, dict): + sub_dict = {} + rec["submodule_hits"] = sub_dict + sh = sub_dict.setdefault(submodule, { + "count": 0, "first_ts": now, "last_ts": now, "tools": [], + }) + sh["count"] += 1 + sh["last_ts"] = now + if tool and tool not in sh["tools"]: + sh["tools"].append(tool) + # 按 agent 维度统计 + if agent: + ag_dict = rec.setdefault("by_agent", {}) + if not isinstance(ag_dict, dict): + ag_dict = {} + rec["by_agent"] = ag_dict + ar = ag_dict.setdefault(agent, { + "count": 0, "first_ts": now, "last_ts": now, + "tools": [], "submodules": [], + }) + ar["count"] += 1 + ar["last_ts"] = now + if tool and tool not in ar["tools"]: + ar["tools"].append(tool) + if submodule and submodule not in ar["submodules"]: + ar["submodules"].append(submodule) + + +def bump_rule( + sess: dict, + name: str, + *, + via: str, + tool: str | None, + meta: dict | None = None, + agent: str | None = None, +) -> None: + """增加 rule 使用计数。""" + if not name: + return + now = time.time() + rec = sess["rules"].setdefault(name, { + "count": 0, + "first_ts": now, + "last_ts": now, + "tools": [], + "via": [], + "source": None, + "by_agent": {}, + }) + rec["count"] += 1 + rec["last_ts"] = now + if tool and tool not in rec["tools"]: + rec["tools"].append(tool) + if via not in rec.get("via", []): + rec.setdefault("via", []).append(via) + if meta and meta.get("source") and not rec.get("source"): + rec["source"] = meta["source"] + if agent: + ag = rec.setdefault("by_agent", {}) + if not isinstance(ag, dict): + ag = {} + rec["by_agent"] = ag + ar = ag.setdefault(agent, {"count": 0, "last_ts": now}) + ar["count"] += 1 + ar["last_ts"] = now + + +def prune_sessions(state: dict, max_sessions: int = 5) -> None: + """在 state 中只保留最近的 N 个 session。""" + # 先过滤掉非 session 键（以下划线开头） + session_keys = [k for k in state if not k.startswith("_")] + if len(session_keys) > max_sessions: + to_remove = sorted( + session_keys, + key=lambda s: state[s].get("started_at", 0) if isinstance(state[s], dict) else 0, + )[:len(session_keys) - max_sessions] + for k in to_remove: + del state[k] + + +def _transcript_scope_key(transcript_path: str | None = None) -> str: + """把 transcript 路径归一化成内部 scope key。""" + path = str(transcript_path or "").strip() + return path or "__session__" + + +def _ensure_transcript_scopes(sess: dict) -> dict[str, dict[str, Any]]: + """确保 session 下存在 transcript scope 容器，并兼容旧字段迁移。""" + scopes = sess.setdefault("_transcripts", {}) + if not isinstance(scopes, dict): + scopes = {} + sess["_transcripts"] = scopes + + legacy_scope = scopes.get("__session__") + if not isinstance(legacy_scope, dict): + legacy_scope = {} + scopes["__session__"] = legacy_scope + + if "offset" not in legacy_scope and "_transcript_offset" in sess: + legacy_scope["offset"] = int(sess.get("_transcript_offset", 0) or 0) + + legacy_usage = sess.get("_last_cumulative_usage") + if "last_cumulative_usage" not in legacy_scope and isinstance(legacy_usage, dict) and legacy_usage: + legacy_scope["last_cumulative_usage"] = legacy_usage + + legacy_dedup = sess.get("_last_usage_event") + if "last_usage_event" not in legacy_scope and isinstance(legacy_dedup, dict) and legacy_dedup: + legacy_scope["last_usage_event"] = legacy_dedup + + return scopes + + +def _get_transcript_scope( + sess: dict, + transcript_path: str | None = None, + *, + create: bool = False, +) -> dict[str, Any] | None: + """获取指定 transcript 的 scope；必要时按需创建。""" + scopes = _ensure_transcript_scopes(sess) + key = _transcript_scope_key(transcript_path) + scope = scopes.get(key) + if isinstance(scope, dict): + return scope + if create: + scope = {} + scopes[key] = scope + return scope + return None + + +def get_transcript_offset(state: dict, sid: str, transcript_path: str | None = None) -> int: + """获取某个 session/transcript 最近一次上报的 transcript 文件 offset。""" + sess = state.get(sid) + if not isinstance(sess, dict): + return 0 + + scope = _get_transcript_scope(sess, transcript_path) + if isinstance(scope, dict): + return int(scope.get("offset", 0) or 0) + + if _transcript_scope_key(transcript_path) == "__session__": + return int(sess.get("_transcript_offset", 0) or 0) + return 0 + + +def set_transcript_offset( + state: dict, + sid: str, + offset: int, + transcript_path: str | None = None, +) -> None: + """在成功上报后持久化 transcript 文件 offset。""" + sess = state.get(sid) + if not isinstance(sess, dict): + return + + scope = _get_transcript_scope(sess, transcript_path, create=True) + if isinstance(scope, dict): + scope["offset"] = int(offset or 0) + + if _transcript_scope_key(transcript_path) == "__session__": + sess["_transcript_offset"] = int(offset or 0) + + +def get_last_cumulative_usage( + state: dict, + sid: str, + transcript_path: str | None = None, +) -> dict[str, int]: + """获取某个 transcript 最近一次上报的累计 token usage。""" + sess = state.get(sid) + if not isinstance(sess, dict): + return {} + + scope = _get_transcript_scope(sess, transcript_path) + if isinstance(scope, dict): + v = scope.get("last_cumulative_usage") + if isinstance(v, dict): + return v + + if _transcript_scope_key(transcript_path) == "__session__": + v = sess.get("_last_cumulative_usage") + if isinstance(v, dict): + return v + return {} + + +def get_last_llm_offset( + state: dict, + sid: str, + transcript_path: str | None = None, +) -> int: + """获取最近一次处理到的 LLM source_offset，用于增量构建 I/O。""" + sess = state.get(sid) + if not isinstance(sess, dict): + return 0 + scope = _get_transcript_scope(sess, transcript_path) + if isinstance(scope, dict): + v = scope.get("last_llm_offset") + if isinstance(v, (int, float)): + return int(v) + return 0 + + +def set_last_llm_offset( + state: dict, + sid: str, + offset: int, + transcript_path: str | None = None, +) -> None: + """持久化某个 transcript 最近处理到的 LLM source_offset。""" + sess = state.get(sid) + if not isinstance(sess, dict): + return + scope = _get_transcript_scope(sess, transcript_path, create=True) + if isinstance(scope, dict): + scope["last_llm_offset"] = int(offset or 0) + + +def set_last_cumulative_usage( + state: dict, + sid: str, + usage: dict[str, int], + transcript_path: str | None = None, +) -> None: + """在处理完一个 turn 后持久化累计 token usage。""" + sess = state.get(sid) + if not isinstance(sess, dict): + return + + scope = _get_transcript_scope(sess, transcript_path, create=True) + if isinstance(scope, dict): + scope["last_cumulative_usage"] = usage + + if _transcript_scope_key(transcript_path) == "__session__": + sess["_last_cumulative_usage"] = usage + + +def get_last_usage_event( + state: dict, + sid: str, + transcript_path: str | None = None, +) -> dict[str, Any] | None: + """获取某个 transcript 最近一次用于去重的 usage 快照。""" + sess = state.get(sid) + if not isinstance(sess, dict): + return None + + scope = _get_transcript_scope(sess, transcript_path) + if isinstance(scope, dict): + event = scope.get("last_usage_event") + if isinstance(event, dict): + return event + + if _transcript_scope_key(transcript_path) == "__session__": + event = sess.get("_last_usage_event") + if isinstance(event, dict): + return event + return None + + +def set_last_usage_event( + state: dict, + sid: str, + usage_event: dict[str, Any], + transcript_path: str | None = None, +) -> None: + """持久化某个 transcript 最近一次用于去重的 usage 快照。""" + sess = state.get(sid) + if not isinstance(sess, dict): + return + + scope = _get_transcript_scope(sess, transcript_path, create=True) + if isinstance(scope, dict): + scope["last_usage_event"] = usage_event + + if _transcript_scope_key(transcript_path) == "__session__": + sess["_last_usage_event"] = usage_event + + +def claim_transcript_event( + state_path: Path, + sid: str, + event_key: str, + transcript_path: str | None = None, + *, + max_seen_keys: int = 512, +) -> bool: + """跨进程原子地声明一个 transcript 作用域内的事件键。""" + seen_path = get_transcript_seen_path(state_path) + scope_key = f"{sid}|{_transcript_scope_key(transcript_path)}" + + with state_lock(state_path): + if seen_path.exists(): + try: + ledger = json.loads(seen_path.read_text("utf-8")) + except Exception: + ledger = {} + else: + ledger = {} + if not isinstance(ledger, dict): + ledger = {} + + seen = ledger.setdefault(scope_key, {}) + if not isinstance(seen, dict): + seen = {} + ledger[scope_key] = seen + if event_key in seen: + return False + seen[event_key] = time.time() + if len(seen) > max_seen_keys: + stale_keys = sorted(seen, key=lambda k: float(seen.get(k, 0) or 0))[:-max_seen_keys] + for key in stale_keys: + seen.pop(key, None) + seen_path.write_text(json.dumps(ledger, ensure_ascii=False), encoding="utf-8") + return True + + +def commit_transcript_progress( + state_path: Path, + sid: str, + transcript_path: str | None = None, + *, + offset: int | None = None, + last_cumulative_usage: dict[str, int] | None = None, +) -> None: + """在处理完一个窗口后，原子地持久化 transcript 解析进度。""" + + def _update(state: dict[str, Any]) -> None: + sess = ensure_session(state, sid) + scope = _get_transcript_scope(sess, transcript_path, create=True) + if not isinstance(scope, dict): + return + if offset is not None: + current_offset = int(scope.get("offset", 0) or 0) + next_offset = int(offset or 0) + if next_offset > current_offset: + scope["offset"] = next_offset + if _transcript_scope_key(transcript_path) == "__session__": + sess["_transcript_offset"] = next_offset + if isinstance(last_cumulative_usage, dict) and last_cumulative_usage: + scope["last_cumulative_usage"] = last_cumulative_usage + if _transcript_scope_key(transcript_path) == "__session__": + sess["_last_cumulative_usage"] = last_cumulative_usage + + update_state_locked(state_path, _update) diff --git a/.codebuddy/skills/agent-observability/scripts/dashboard/index.html b/.codebuddy/skills/agent-observability/scripts/dashboard/index.html new file mode 100644 index 0000000..8c391f5 --- /dev/null +++ b/.codebuddy/skills/agent-observability/scripts/dashboard/index.html @@ -0,0 +1,1086 @@ + + + + + +Agent Observability + + + + + +
+
+
+
Agent Observability
+
读取本地 dashboard-data.json（由 build_dashboard_data.py 从 metrics.ndjson / workflow-state.json 聚合而成）· LoopForge 项目
+
+
加载中…
+
+ +
+ 时间范围 + + + +
+ +
+ +
+ + + + + +
+ +
+
+
建议关注按当前范围内的阈值规则实时算出，不是固定文案
+
+
+ +
+
+ 每日成本 +
cost_usd（按 config/pricing.json 估算）
+
+
+
+ +
+
+
+ Token 构成 +
+
+
+
+
+
按模型的成本占比Top 4 + 其他
+
+
+
+ +
+
数据表图表的完整数值备份
+
日期会话数成本 (USD)inputoutputcache_read工具失败
+
+
+ + + + + + + + + +
+ 数据来源：python3 ../build_dashboard_data.py --project-root <devflow 项目根目录> 会读取该项目下 + .codebuddy/skills/agent-observability/logs/metrics.ndjson、.state.json、 + artifacts/*/workflow-state.json（Classic/Portable 两种 schema 自动识别）与 + .codebuddy/hooks/logs/auto-dispatch.log，聚合成同目录下的 dashboard-data.json。 + 这个页面只读那份快照，不直接碰任何日志/状态文件——重新生成快照后刷新页面即可看到最新数据。 + 全部处理在本机完成，不发往任何外部服务。 +
+
+ + + + diff --git a/.codebuddy/skills/agent-observability/scripts/main.py b/.codebuddy/skills/agent-observability/scripts/main.py new file mode 100644 index 0000000..4ca2481 --- /dev/null +++ b/.codebuddy/skills/agent-observability/scripts/main.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +"""agent-observability 的 hook 入口。""" +from __future__ import annotations + +import sys +from pathlib import Path + + +if __package__ in (None, ""): + _SCRIPTS_DIR = Path(__file__).resolve().parent + if str(_SCRIPTS_DIR) not in sys.path: + sys.path.insert(0, str(_SCRIPTS_DIR)) + from core.runtime import main as runtime_main # type: ignore +else: + from .core.runtime import main as runtime_main + + +def main() -> int: + return runtime_main(sys.argv[1:]) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.codebuddy/skills/agent-observability/scripts/run_hook.sh b/.codebuddy/skills/agent-observability/scripts/run_hook.sh new file mode 100755 index 0000000..d476810 --- /dev/null +++ b/.codebuddy/skills/agent-observability/scripts/run_hook.sh @@ -0,0 +1,16 @@ +#!/bin/sh +set -eu + +PROJECT_DIR="${CODEBUDDY_PROJECT_DIR:-$(cd "$(dirname "$0")"/../../../.. && pwd -P)}" +PYTHON_BIN="$PROJECT_DIR/.venv/bin/python" +MAIN_PY="$PROJECT_DIR/.codebuddy/skills/agent-observability/scripts/main.py" + +# 只有当 .venv 里可用 zhiyanllm 时才优先使用该解释器； +# 否则回退到已安装 zhiyanllm 的系统 python3。 +if [ -x "$PYTHON_BIN" ]; then + if "$PYTHON_BIN" -c "import zhiyanllm" 2>/dev/null; then + exec "$PYTHON_BIN" "$MAIN_PY" "$@" + fi +fi + +exec python3 "$MAIN_PY" "$@" diff --git a/.codebuddy/skills/agent-observability/templates/settings-hook.json b/.codebuddy/skills/agent-observability/templates/settings-hook.json new file mode 100644 index 0000000..7646ee7 --- /dev/null +++ b/.codebuddy/skills/agent-observability/templates/settings-hook.json @@ -0,0 +1,62 @@ +{ + "_comment": "将本文件合并到 ~/.codebuddy/settings.json 或 /.codebuddy/settings.json", + "hooks": { + "SessionStart": [ + { + "hooks": [ + { + "type": "command", + "command": "\"$CODEBUDDY_PROJECT_DIR\"/.codebuddy/skills/agent-observability/scripts/run_hook.sh session-start", + "timeout": 5 + } + ] + } + ], + "UserPromptSubmit": [ + { + "hooks": [ + { + "type": "command", + "command": "\"$CODEBUDDY_PROJECT_DIR\"/.codebuddy/skills/agent-observability/scripts/run_hook.sh user-prompt-submit", + "timeout": 5 + } + ] + } + ], + "PreToolUse": [ + { + "matcher": "", + "hooks": [ + { + "type": "command", + "command": "\"$CODEBUDDY_PROJECT_DIR\"/.codebuddy/skills/agent-observability/scripts/run_hook.sh pre", + "timeout": 5 + } + ] + } + ], + "PostToolUse": [ + { + "matcher": "", + "hooks": [ + { + "type": "command", + "command": "\"$CODEBUDDY_PROJECT_DIR\"/.codebuddy/skills/agent-observability/scripts/run_hook.sh post", + "timeout": 30 + } + ] + } + ], + "Stop": [ + { + "hooks": [ + { + "type": "command", + "command": "\"$CODEBUDDY_PROJECT_DIR\"/.codebuddy/skills/agent-observability/scripts/run_hook.sh stop", + "timeout": 30 + } + ] + } + ] + } +} diff --git a/.codebuddy/skills/agent-observability/tests/fixtures/transcript.jsonl b/.codebuddy/skills/agent-observability/tests/fixtures/transcript.jsonl new file mode 100644 index 0000000..a5ea47d --- /dev/null +++ b/.codebuddy/skills/agent-observability/tests/fixtures/transcript.jsonl @@ -0,0 +1,10 @@ +{"type":"system","subtype":"init","session_id":"demo-001","content":"\ndo not exec\n\n\npdf\nhermes-devflow\n"} +{"type":"assistant","providerData":{"messageId":"msg-001"},"content":[{"type":"tool_use","name":"read_file","input":{"filePath":"README.md"}}],"usage":{"input_tokens":1820,"output_tokens":42,"cache_read_input_tokens":15300}} +{"type":"function_call","name":"Read","callId":"call-001","providerData":{"messageId":"msg-001"}} +{"type":"function_call_result","name":"Read","callId":"call-001","providerData":{"messageId":"msg-001"}} +{"type":"assistant","providerData":{"messageId":"msg-002"},"content":[{"type":"tool_use","name":"use_skill","input":{"command":"pdf"}}],"usage":{"input_tokens":2100,"output_tokens":58,"cache_read_input_tokens":15300}} +{"type":"function_call","name":"Bash","callId":"call-002","providerData":{"messageId":"msg-002"}} +{"type":"function_call","name":"Bash","callId":"call-003","providerData":{"messageId":"msg-002"}} +{"type":"function_call_result","name":"Bash","callId":"call-002","providerData":{"messageId":"msg-002"}} +{"type":"function_call_result","name":"Bash","callId":"call-003","providerData":{"messageId":"msg-002"}} +{"type":"assistant","providerData":{"messageId":"msg-003"},"content":"完成","usage":{"input_tokens":2240,"output_tokens":120,"cache_read_input_tokens":15300}} diff --git a/.codebuddy/skills/agent-observability/tests/test_agent_identity.py b/.codebuddy/skills/agent-observability/tests/test_agent_identity.py new file mode 100644 index 0000000..664cc1d --- /dev/null +++ b/.codebuddy/skills/agent-observability/tests/test_agent_identity.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path +import sys + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "scripts" +if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) + +from core.agent_identity import AgentIdentityResolver # type: ignore +from core import state as st # type: ignore + + +class AgentIdentityResolverTests(unittest.TestCase): + def test_extract_identity_from_send_message_json(self): + resolver = AgentIdentityResolver() + data = { + "tool_name": "send_message", + "tool_input": { + "content": '{"from_role":"developer","next_target":{"role_name":"leader"}}' + }, + } + current, dispatched = resolver.extract_identity(data) + self.assertEqual(current, "developer") + self.assertEqual(dispatched, "leader") + + def test_track_returns_active_and_dispatched(self): + resolver = AgentIdentityResolver() + with tempfile.TemporaryDirectory() as td: + state_path = Path(td) / ".state.json" + sid = "s2" + state_data = st.load_state(state_path) + st.ensure_session(state_data, sid) + st.save_state(state_path, state_data) + + data = { + "tool_name": "Task", + "tool_input": {"subagent_name": "developer"}, + } + active, dispatched = resolver.track(state_path=state_path, sid=sid, data=data) + self.assertEqual(active, "main") + self.assertEqual(dispatched, "developer") + + state2 = st.load_state(state_path) + sess = state2.get(sid, {}) + self.assertEqual(sess.get("current_agent"), "developer") + self.assertEqual(sess.get("dispatched", {}).get("developer"), 1) + + def test_track_recognizes_agent_tool_subagent_type(self): + """Claude Code 的 Agent 工具用 subagent_type（不是 CodeBuddy 原生 Task 的 + subagent_name）标识派发目标；这条路径之前没有测试覆盖，_extract_from_task + 漏认这个字段会导致 dispatch 统计对所有 Agent 工具派发的场景失明。""" + resolver = AgentIdentityResolver() + with tempfile.TemporaryDirectory() as td: + state_path = Path(td) / ".state.json" + sid = "s3" + state_data = st.load_state(state_path) + st.ensure_session(state_data, sid) + st.save_state(state_path, state_data) + + data = { + "tool_name": "Agent", + "tool_input": {"subagent_type": "worker", "description": "spawn a team"}, + } + active, dispatched = resolver.track(state_path=state_path, sid=sid, data=data) + self.assertEqual(active, "main") + self.assertEqual(dispatched, "worker") + + state2 = st.load_state(state_path) + sess = state2.get(sid, {}) + self.assertEqual(sess.get("dispatched", {}).get("worker"), 1) + hist = sess.get("agent_history") or [] + self.assertTrue(any( + h.get("agent") == "worker" and str(h.get("evidence") or "").startswith("dispatch") + for h in hist + )) + + def test_extract_from_task_prefers_subagent_type_over_subagent_name(self): + resolver = AgentIdentityResolver() + result = resolver._extract_from_task({"subagent_type": "explorer", "subagent_name": "developer"}) + self.assertEqual(result, "explorer") diff --git a/.codebuddy/skills/agent-observability/tests/test_build_dashboard_data.py b/.codebuddy/skills/agent-observability/tests/test_build_dashboard_data.py new file mode 100644 index 0000000..53b031c --- /dev/null +++ b/.codebuddy/skills/agent-observability/tests/test_build_dashboard_data.py @@ -0,0 +1,603 @@ +from __future__ import annotations + +import contextlib +import io +import json +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "scripts" +if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) + +import build_dashboard_data as bdd # type: ignore + + +class ToolCallFailedTests(unittest.TestCase): + """真实 CodeBuddy CLI transcript 里从没出现过 `is_error` 这个键——实测抓到的 + rawResponse 形如 {"exitCode": 0, "tool_error_code": "0", ...}（成功）或非 0 + exitCode（失败）。之前只认 is_error，导致这个宿主上失败统计永远是 0。""" + + def test_real_success_shape_from_codebuddy_is_not_a_failure(self): + # 实测数据：ls 一个不存在的路径，但命令写成了 `ls ...; echo "EXIT_CODE=$?"`， + # 复合命令整体 exitCode 被内层 echo 冲成了 0——工具调用本身没有失败。 + raw_response = { + "exitCode": 0, "signal": None, "interrupted": False, + "sandboxDenied": False, "stderrBytesTruncated": 0, + "stdoutBytesTruncated": 0, "tool_error_code": "0", + } + self.assertFalse(bdd.tool_call_failed(raw_response)) + + def test_nonzero_exit_code_is_a_failure(self): + raw_response = {"exitCode": 1, "tool_error_code": "1"} + self.assertTrue(bdd.tool_call_failed(raw_response)) + + def test_nonzero_tool_error_code_without_exit_code_is_a_failure(self): + raw_response = {"tool_error_code": "127"} + self.assertTrue(bdd.tool_call_failed(raw_response)) + + def test_legacy_is_error_flag_still_recognized(self): + self.assertTrue(bdd.tool_call_failed({"is_error": True})) + + def test_missing_or_non_dict_raw_response_is_not_a_failure(self): + self.assertFalse(bdd.tool_call_failed(None)) + self.assertFalse(bdd.tool_call_failed("not a dict")) + self.assertFalse(bdd.tool_call_failed({})) + + def test_build_failures_and_session_status_use_exit_code(self): + events = [ + { + "event": "tool", "sid": "s1", "ts": 1.0, "tool": "Bash", + "tool_details": {"raw_response": {"exitCode": 1, "tool_error_code": "1"}}, + }, + ] + failures = bdd.build_failures(events) + self.assertEqual(len(failures), 1) + self.assertEqual(failures[0]["tool"], "Bash") + self.assertEqual(failures[0]["code"], "1") + + _, sessions = bdd.build_daily_and_sessions(events) + self.assertEqual(sessions[0]["status"], "error") + + +class ResolveInputPathsTests(unittest.TestCase): + def setUp(self): + self.skill_root = Path("/tmp/fake-skill-root") + + def test_defaults_fall_back_to_skill_root_logs(self): + metrics, state = bdd.resolve_input_paths(self.skill_root) + self.assertEqual(metrics, self.skill_root / "logs" / "metrics.ndjson") + self.assertEqual(state, self.skill_root / "logs" / ".state.json") + + def test_none_args_equivalent_to_empty_args(self): + m1, s1 = bdd.resolve_input_paths(self.skill_root, None, None) + m2, s2 = bdd.resolve_input_paths(self.skill_root, "", "") + self.assertEqual((m1, s1), (m2, s2)) + + def test_both_args_override(self): + metrics, state = bdd.resolve_input_paths( + self.skill_root, "/abs/a/metrics.ndjson", "/abs/b/.state.json" + ) + self.assertEqual(metrics, Path("/abs/a/metrics.ndjson").absolute()) + self.assertEqual(state, Path("/abs/b/.state.json").absolute()) + + def test_metrics_only_override_keeps_default_state(self): + metrics, state = bdd.resolve_input_paths( + self.skill_root, "/abs/a/metrics.ndjson", None + ) + self.assertEqual(metrics, Path("/abs/a/metrics.ndjson").absolute()) + self.assertEqual(state, self.skill_root / "logs" / ".state.json") + + def test_state_only_override_keeps_default_metrics(self): + metrics, state = bdd.resolve_input_paths( + self.skill_root, None, "/abs/b/.state.json" + ) + self.assertEqual(metrics, self.skill_root / "logs" / "metrics.ndjson") + self.assertEqual(state, Path("/abs/b/.state.json").absolute()) + + def test_tilde_expansion(self): + metrics, state = bdd.resolve_input_paths( + self.skill_root, "~/x/metrics.ndjson", "~/y/.state.json" + ) + self.assertEqual(metrics, (Path.home() / "x" / "metrics.ndjson").absolute()) + self.assertEqual(state, (Path.home() / "y" / ".state.json").absolute()) + + +class MainEndToEndTests(unittest.TestCase): + def _run_main(self, argv: list[str]) -> None: + old_argv = sys.argv + sys.argv = ["build_dashboard_data.py"] + argv + try: + self.assertEqual(bdd.main(), 0) + finally: + sys.argv = old_argv + + def test_default_paths_unchanged_and_reflected_in_source(self): + with tempfile.TemporaryDirectory() as td: + out = Path(td) / "dashboard-data.json" + self._run_main(["--project-root", td, "--out", str(out)]) + self.assertTrue(out.is_file()) + data = json.loads(out.read_text("utf-8")) + expected_metrics = str(bdd.SCRIPTS_DIR.parent / "logs" / "metrics.ndjson") + expected_state = str(bdd.SCRIPTS_DIR.parent / "logs" / ".state.json") + self.assertEqual(data["source"]["metrics_ndjson"], expected_metrics) + self.assertEqual(data["source"]["state_json"], expected_state) + + def test_overridden_paths_read_and_reflected_in_source(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + metrics_file = root / "metrics.ndjson" + state_file = root / ".state.json" + metrics_file.write_text( + '{"event":"usage","sid":"s1","ts":1,"tokens":{"input":10,"output":20}}\n', + encoding="utf-8", + ) + state_file.write_text(json.dumps({"s1": {"skills": {}}}), encoding="utf-8") + out = root / "dashboard-data.json" + + self._run_main([ + "--project-root", td, + "--out", str(out), + "--metrics-path", str(metrics_file), + "--state-path", str(state_file), + ]) + self.assertTrue(out.is_file()) + + data = json.loads(out.read_text("utf-8")) + # resolve_input_paths 对覆盖路径执行 .expanduser().absolute()，断言需对齐（macOS 下不用 .resolve() 以避免 /tmp → /private/tmp 符号链接错位）。 + self.assertEqual(data["source"]["metrics_ndjson"], str(metrics_file.absolute())) + self.assertEqual(data["source"]["state_json"], str(state_file.absolute())) + # 覆盖文件确实被读取：metrics 行被解析进了 event_count。 + self.assertEqual(data["source"]["event_count"], 1) + + +class TopSlowTests(unittest.TestCase): + """`--top-slow N` 是纯增量能力：默认不开启，开启后也只往终端打印，绝不改动 + dashboard-data.json 的结构（看板前端按字段取值，多一个字段或少一个字段都会 + 直接影响渲染）。这些用例把"降序取前 N""非 tool 事件排除""ms 兜底""只读" + 四条边界钉住，避免后续有人顺手把结果写进 JSON 或改了排序方向。""" + + def _events(self): + return [ + {"event": "user_prompt_submit", "sid": "s1", "ts": 1.0, "turn_id": "t1", "agent": "main"}, + {"event": "tool", "sid": "s1", "ts": 2.0, "tool": "Read", "ms": 300, "agent": "main"}, + {"event": "usage", "sid": "s1", "ts": 3.0, "tokens": {"input": 10, "output": 5}, "agent": "main"}, + {"event": "tool", "sid": "s1", "ts": 4.0, "tool": "Bash", "ms": 900, "agent": "main"}, + {"event": "tool", "sid": "s1", "ts": 5.0, "tool": "Grep", "ms": 120, "agent": "main"}, + ] + + def _write_metrics(self, root: Path) -> Path: + metrics = root / "metrics.ndjson" + metrics.write_text( + "\n".join(json.dumps(e) for e in self._events()) + "\n", + encoding="utf-8", + ) + return metrics + + def _run_main(self, argv: list[str]) -> str: + """跑一次 main() 并返回捕获到的 stdout——top-slow 的产出只体现在终端上， + 不体现在 JSON 里，所以断言必须落在 stdout。""" + old_argv = sys.argv + sys.argv = ["build_dashboard_data.py"] + argv + buf = io.StringIO() + try: + with contextlib.redirect_stdout(buf): + rc = bdd.main() + finally: + sys.argv = old_argv + self.assertEqual(rc, 0) + return buf.getvalue() + + def _printed_tools(self, stdout: str) -> list[str]: + tools = [] + for line in stdout.splitlines(): + line = line.strip() + if not line or not line[0].isdigit(): + continue + tools.append(line.split(". ", 1)[1].split()[0]) + return tools + + def test_returns_n_slowest_tool_events_descending(self): + rows = bdd.build_top_slow(self._events(), 2) + self.assertEqual([(r["tool"], r["ms"]) for r in rows], [("Bash", 900), ("Read", 300)]) + + def test_limit_greater_than_event_count_returns_all_without_error(self): + events = [{"event": "tool", "sid": "s1", "ts": 1.0, "tool": "Bash", "ms": 10}] + rows = bdd.build_top_slow(events, 99) + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["tool"], "Bash") + + def test_non_tool_events_are_excluded(self): + rows = bdd.build_top_slow(self._events(), 10) + self.assertEqual([r["tool"] for r in rows], ["Bash", "Read", "Grep"]) + # usage 事件即使带 ms 也不能混进来。 + self.assertTrue(all(isinstance(r["ms"], (int, float)) for r in rows)) + + def test_missing_or_non_numeric_ms_falls_back_to_zero(self): + events = [ + {"event": "tool", "sid": "s1", "ts": 1.0, "tool": "Bash"}, + {"event": "tool", "sid": "s1", "ts": 2.0, "tool": "Read", "ms": "120"}, + {"event": "tool", "sid": "s1", "ts": 3.0, "tool": "Grep", "ms": None}, + {"event": "tool", "sid": "s1", "ts": 4.0, "tool": "Edit", "ms": 1}, + ] + rows = bdd.build_top_slow(events, 4) + self.assertEqual(len(rows), 4) + # 非数字/缺失的 ms 一律按 0 计，不会把字符串 "120" 排到最前面。 + self.assertEqual(sum(r["ms"] for r in rows), 1) + + def test_disabled_by_default_prints_nothing_extra(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + metrics = self._write_metrics(root) + out = root / "dashboard-data.json" + stdout = self._run_main([ + "--project-root", td, "--out", str(out), + "--metrics-path", str(metrics), "--state-path", str(root / ".state.json"), + ]) + self.assertIn("wrote ", stdout) + self.assertNotIn("top-slow", stdout) + self.assertEqual(self._printed_tools(stdout), []) + + def test_enabled_prints_tool_and_ms_descending(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + metrics = self._write_metrics(root) + out = root / "dashboard-data.json" + stdout = self._run_main([ + "--project-root", td, "--out", str(out), + "--metrics-path", str(metrics), "--state-path", str(root / ".state.json"), + "--top-slow", "2", + ]) + self.assertIn("wrote ", stdout) + self.assertIn("top-slow", stdout) + self.assertEqual(self._printed_tools(stdout), ["Bash", "Read"]) + self.assertIn("900ms", stdout) + self.assertNotIn("120ms", stdout) # N=2，第三名的 Grep 不该出现 + + def test_enabled_does_not_change_dashboard_data_structure(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + metrics = self._write_metrics(root) + state = root / ".state.json" + state.write_text(json.dumps({"s1": {"skills": {}}}), encoding="utf-8") + # 输出到两个不同文件再比对，避免"第二次跑覆盖了第一次"导致对比失真。 + out_on = root / "on.json" + out_off = root / "off.json" + self._run_main([ + "--project-root", td, "--out", str(out_on), + "--metrics-path", str(metrics), "--state-path", str(state), "--top-slow", "5", + ]) + self._run_main([ + "--project-root", td, "--out", str(out_off), + "--metrics-path", str(metrics), "--state-path", str(state), + ]) + data_on = json.loads(out_on.read_text("utf-8")) + data_off = json.loads(out_off.read_text("utf-8")) + data_on.pop("generated_at") + data_off.pop("generated_at") + self.assertEqual(data_on, data_off) + # 兜底断言：结果没有以任何形式渗进 JSON（不只是"结构相同"）。 + self.assertNotIn("topSlow", json.dumps(data_on)) + self.assertNotIn("top-slow", json.dumps(data_on)) + + +class BuildDailyAndSessionsTurnAttributionTests(unittest.TestCase): + """真实场景复现过：AgentLens（写 state.current_turn 的那套 tracing）关闭时， + tool/usage 事件自己的 turn_id 永远是 None，只有 user_prompt_submit 事件带真实 + turn_id。修复前所有没有 turn_id 的事件会被塌缩进同一个 "?" 占位桶，看起来 + 像"整个会话只有 1 个 turn"，哪怕实际发了好几轮 prompt。""" + + def _events(self): + return [ + {"event": "user_prompt_submit", "sid": "s1", "ts": 1.0, "turn_id": "t1", "agent": "main"}, + {"event": "tool", "sid": "s1", "ts": 2.0, "tool": "Bash", "ms": 100, "agent": "main"}, + {"event": "usage", "sid": "s1", "ts": 3.0, "tokens": {"input": 10, "output": 5}, "agent": "main"}, + {"event": "user_prompt_submit", "sid": "s1", "ts": 4.0, "turn_id": "t2", "agent": "main"}, + {"event": "tool", "sid": "s1", "ts": 5.0, "tool": "Read", "ms": 50, "agent": "main"}, + {"event": "tool", "sid": "s1", "ts": 6.0, "tool": "Edit", "ms": 80, "agent": "main"}, + ] + + def test_turns_reflect_prompt_boundaries_not_a_single_bucket(self): + daily, sessions = bdd.build_daily_and_sessions(self._events()) + self.assertEqual(len(sessions), 1) + sess = sessions[0] + self.assertEqual(sess["turns"], 2) + self.assertEqual(sess["toolCalls"], 3) + self.assertEqual(len(sess["timeline"]), 2) + turn1_tools = [e["tool"] for e in sess["timeline"][0]["events"] if e["kind"] == "tool"] + turn2_tools = [e["tool"] for e in sess["timeline"][1]["events"] if e["kind"] == "tool"] + self.assertEqual(turn1_tools, ["Bash"]) + self.assertEqual(turn2_tools, ["Read", "Edit"]) + + def test_events_before_first_prompt_fall_back_to_unknown_bucket(self): + events = [ + {"event": "tool", "sid": "s1", "ts": 1.0, "tool": "Bash", "ms": 100, "agent": "main"}, + {"event": "user_prompt_submit", "sid": "s1", "ts": 2.0, "turn_id": "t1", "agent": "main"}, + {"event": "tool", "sid": "s1", "ts": 3.0, "tool": "Read", "ms": 50, "agent": "main"}, + ] + _, sessions = bdd.build_daily_and_sessions(events) + sess = sessions[0] + # 没有归属到任何真实 turn 的事件仍然单独成桶，不会被错误地并入第一个真实 turn。 + self.assertEqual(len(sess["timeline"]), 2) + self.assertEqual(sess["timeline"][0]["turn"], 1) + + def test_prompt_only_turn_with_no_tool_calls_still_appears_in_timeline(self): + """真实数据复现过：纯对话轮次（用户发了 prompt，但助手没有调用任何工具， + 也没有产生 usage 事件）之前完全不会在 timeline 里建桶——turns 计数是对的 + （来自 user_prompt_submit 自带的 turn_id），但展开的时间线分组数会比 turns + 少，二者对不上。""" + events = [ + {"event": "user_prompt_submit", "sid": "s1", "ts": 1.0, "turn_id": "t1", "agent": "main"}, + {"event": "tool", "sid": "s1", "ts": 2.0, "tool": "Read", "ms": 100, "agent": "main"}, + {"event": "user_prompt_submit", "sid": "s1", "ts": 3.0, "turn_id": "t2", "agent": "main"}, + {"event": "user_prompt_submit", "sid": "s1", "ts": 4.0, "turn_id": "t3", "agent": "main"}, + {"event": "tool", "sid": "s1", "ts": 5.0, "tool": "Grep", "ms": 50, "agent": "main"}, + ] + _, sessions = bdd.build_daily_and_sessions(events) + sess = sessions[0] + self.assertEqual(sess["turns"], 3) + self.assertEqual(len(sess["timeline"]), 3) + self.assertEqual(sess["timeline"][1]["events"], []) + + def test_prompt_only_session_still_counted_in_daily_session_count(self): + """真实数据复现过（TC5 当天 5 个 session，"总览"页汇总出的会话数却是 4）： + daily["_sids"]（"会话浏览"总览卡片"会话数"的数据源）之前只在 tool/usage + 分支里 add，一个全程只发了 prompt、没有触发任何工具调用也没有 usage 事件 + 的 session（比如用户发了消息但被拒绝/打断，没有真正执行）会被这天的 + sessionCount 完全漏掉——尽管 sessions 列表（"会话浏览"tab）里它确实在， + 导致总览页"会话数"比实际能展开看到的会话数少。""" + events = [ + {"event": "user_prompt_submit", "sid": "s1", "ts": 1.0, "turn_id": "t1", "agent": "main"}, + {"event": "tool", "sid": "s1", "ts": 2.0, "tool": "Read", "ms": 50, "agent": "main"}, + {"event": "user_prompt_submit", "sid": "s2", "ts": 3.0, "turn_id": "t1", "agent": "main"}, + ] + daily, sessions = bdd.build_daily_and_sessions(events) + self.assertEqual(len(sessions), 2) + self.assertEqual(len(daily), 1) + self.assertEqual(daily[0]["sessionCount"], 2) + + +class BuildDevflowRunsClassicSoloTests(unittest.TestCase): + """真实运行验证过：Classic small 任务会经过 PHASE-0 -> SOLO -> TASK-05 + （knowledge 由 solo-developer 合并执行），不是只有 PHASE-0/SOLO 两步。""" + + def test_solo_run_includes_task05_and_marks_phase0_completed(self): + with tempfile.TemporaryDirectory() as td: + project_root = Path(td) + run_dir = project_root / "artifacts" / "solo-with-knowledge_20260914_0600" + run_dir.mkdir(parents=True) + (run_dir / "workflow-state.json").write_text(json.dumps({ + "version": "1.3", + "task_slug": "solo-with-knowledge_20260914_0600", + "size_class": "small", + "current_stage": "SUMMARY", + "last_event": "workflow_completed", + # 注意：真实 schema 里 stages{} 不含 PHASE-0 —— Phase 0 是隐式完成的。 + "stages": { + "SOLO": {"status": "completed", "executor": "solo-developer", "retry_count": 0}, + "TASK-02": {"status": "skipped", "executor": "architect", "retry_count": 0}, + "TASK-03": {"status": "skipped", "executor": "developer", "retry_count": 0}, + "CODE-REVIEW": {"status": "skipped", "executor": "code-reviewer", "retry_count": 0}, + "TASK-04": {"status": "skipped", "executor": "test-engineer", "retry_count": 0}, + "TASK-05": {"status": "completed", "executor": "solo-developer", "retry_count": 0}, + }, + }), encoding="utf-8") + + runs = bdd.build_devflow_runs(project_root) + self.assertEqual(len(runs), 1) + stage_by_key = {s["key"]: s["status"] for s in runs[0]["stages"]} + self.assertIn("TASK-05", stage_by_key) + self.assertEqual(stage_by_key["TASK-05"], "completed") + self.assertEqual(stage_by_key["PHASE-0"], "completed") + self.assertNotIn("TASK-01", stage_by_key) # SOLO 路径不该混进 medium/large 的阶段 + + +class BuildDevflowRunsMediumWithLingeringSoloKeyTests(unittest.TestCase): + """用户反馈复现：workflow-state.json 模板固定给每次运行都写一份 SOLO stage + （见 assets/workflow-state-template.json 里 SOLO.description 的说明： + "仅当 size_class==small 时启用，否则保持 skipped"），medium/large 任务从不会 + 真正执行它，状态停在 "pending" 或 "skipped"。修复前 `"SOLO" in stages` 只看 + key 存不存在，不看状态，导致这类 medium 任务被误判成 solo 路径，阶段视图会用 + CLASSIC_SOLO_ORDER 渲染，把 TASK-01/TASK-02/TASK-03/CODE-REVIEW/TASK-04 全部 + 从 stepper 里漏掉。""" + + def _run(self, solo_status: str): + with tempfile.TemporaryDirectory() as td: + project_root = Path(td) + run_dir = project_root / "artifacts" / "medium-with-lingering-solo_20260915_0900" + run_dir.mkdir(parents=True) + (run_dir / "workflow-state.json").write_text(json.dumps({ + "version": "1.3", + "task_slug": "medium-with-lingering-solo_20260915_0900", + "size_class": "medium", + "current_stage": "TASK-03", + "last_event": "TASK-02_completed", + "stages": { + "SOLO": {"status": solo_status, "executor": "solo-developer", "retry_count": 0}, + "TASK-01": {"status": "completed", "executor": "main", "retry_count": 0}, + "TASK-02": {"status": "completed", "executor": "architect", "retry_count": 0}, + "TASK-03": {"status": "in_progress", "executor": "developer", "retry_count": 0}, + "CODE-REVIEW": {"status": "pending", "executor": "code-reviewer", "retry_count": 0}, + "TASK-04": {"status": "pending", "executor": "test-engineer", "retry_count": 0}, + "TASK-05": {"status": "pending", "executor": "knowledge-engineer", "retry_count": 0}, + }, + }), encoding="utf-8") + return bdd.build_devflow_runs(project_root) + + def test_medium_run_with_skipped_solo_key_uses_full_classic_order(self): + runs = self._run("skipped") + self.assertEqual(len(runs), 1) + stage_by_key = {s["key"]: s["status"] for s in runs[0]["stages"]} + # 修复前：这里会走 CLASSIC_SOLO_ORDER，TASK-01/02/03/CODE-REVIEW/TASK-04 全部消失。 + self.assertIn("TASK-01", stage_by_key) + self.assertIn("TASK-02", stage_by_key) + self.assertIn("TASK-03", stage_by_key) + self.assertIn("CODE-REVIEW", stage_by_key) + self.assertIn("TASK-04", stage_by_key) + self.assertEqual(stage_by_key["TASK-02"], "completed") + self.assertEqual(stage_by_key["TASK-03"], "in_progress") + + def test_medium_run_with_pending_solo_key_uses_full_classic_order(self): + # 模板初始状态是 "pending"（不是所有实现都会显式改成 "skipped"），同样不该被判成 solo。 + runs = self._run("pending") + stage_by_key = {s["key"]: s["status"] for s in runs[0]["stages"]} + self.assertIn("TASK-01", stage_by_key) + self.assertIn("CODE-REVIEW", stage_by_key) + + +class BuildDevflowRunsDurationTests(unittest.TestCase): + """真实数据复现过：TASK-05 的 started_at 可能早于 SOLO 的 completed_at + （同一次 solo-developer 执行内部的子步骤，不是真正先后发生的两个阶段）。 + total_duration 必须按真实起止跨度算，不能对逐阶段 duration 求和——否则会把 + 重叠部分重复计入，虚高于源数据本身反映的运行时长（这次修复前是 600s， + 真实跨度只有 540s）。""" + + def _write_state(self, project_root: Path, stages: dict) -> None: + run_dir = project_root / "artifacts" / "overlap-run_20260914_0620" + run_dir.mkdir(parents=True) + (run_dir / "workflow-state.json").write_text(json.dumps({ + "version": "1.3", + "task_slug": "overlap-run_20260914_0620", + "size_class": "small", + "current_stage": "SUMMARY", + "last_event": "workflow_completed", + "stages": stages, + }), encoding="utf-8") + + def test_total_duration_uses_true_span_not_sum_of_overlapping_stages(self): + with tempfile.TemporaryDirectory() as td: + project_root = Path(td) + self._write_state(project_root, { + "SOLO": { + "status": "completed", "executor": "solo-developer", "retry_count": 0, + "started_at": "2026-09-14T06:20:00Z", "completed_at": "2026-09-14T06:29:00Z", + }, + "TASK-05": { + "status": "completed", "executor": "solo-developer", "retry_count": 0, + # 早于 SOLO 的 completed_at —— 真实数据里观测到的重叠场景。 + "started_at": "2026-09-14T06:28:00Z", "completed_at": "2026-09-14T06:29:00Z", + }, + }) + runs = bdd.build_devflow_runs(project_root) + self.assertEqual(len(runs), 1) + # 真实跨度是 06:20~06:29 = 540s，不是 540+60=600s。 + self.assertEqual(runs[0]["duration"], 540) + stage_by_key = {s["key"]: s["duration"] for s in runs[0]["stages"]} + # 单个阶段自身的 duration 不受影响，仍然如实反映各自的起止跨度。 + self.assertEqual(stage_by_key["SOLO"], 540) + self.assertEqual(stage_by_key["TASK-05"], 60) + + def test_total_duration_is_zero_when_no_stage_has_valid_timestamps(self): + with tempfile.TemporaryDirectory() as td: + project_root = Path(td) + self._write_state(project_root, { + "SOLO": {"status": "completed", "executor": "solo-developer", "retry_count": 0}, + "TASK-05": {"status": "completed", "executor": "solo-developer", "retry_count": 0}, + }) + runs = bdd.build_devflow_runs(project_root) + self.assertEqual(runs[0]["duration"], 0) + + +class BuildDailyAndSessionsPerEventAgentTests(unittest.TestCase): + """真实数据复现过（TC5，size_class=small 但走了真实 team_create/send_message + 派发）：同一个 sid 下，metrics.ndjson 的 tool/usage 事件本来就各自带着准确的 + agent 字段（一次真实会话里 66 条 main、64 条 solo-developer），但展开进 + timeline 的每条 event 字典只有 kind/tool/ms/err，从不写回 agent —— 时间线里 + 完全看不出某次工具调用到底是 main 自己做的还是派发给 solo-developer 后做的， + 等于白白丢弃了上游已经采集到的数据。""" + + def _events(self): + return [ + {"event": "user_prompt_submit", "sid": "s1", "ts": 1.0, "turn_id": "t1", "agent": "main"}, + {"event": "tool", "sid": "s1", "ts": 2.0, "tool": "Agent", "ms": 50, "agent": "main"}, + {"event": "tool", "sid": "s1", "ts": 3.0, "tool": "Bash", "ms": 100, "agent": "solo-developer"}, + {"event": "usage", "sid": "s1", "ts": 4.0, "tokens": {"input": 10, "output": 5}, "agent": "solo-developer"}, + {"event": "tool", "sid": "s1", "ts": 5.0, "tool": "SendMessage", "ms": 30, "agent": "main"}, + ] + + def test_timeline_events_carry_the_agent_that_actually_ran_them(self): + _, sessions = bdd.build_daily_and_sessions(self._events()) + sess = sessions[0] + events = sess["timeline"][0]["events"] + by_tool = {e.get("tool"): e for e in events if e["kind"] == "tool"} + self.assertEqual(by_tool["Agent"]["agent"], "main") + self.assertEqual(by_tool["Bash"]["agent"], "solo-developer") + self.assertEqual(by_tool["SendMessage"]["agent"], "main") + usage_events = [e for e in events if e["kind"] == "usage"] + self.assertEqual(usage_events[0]["agent"], "solo-developer") + + def test_missing_agent_on_raw_event_falls_back_to_main(self): + events = [ + {"event": "user_prompt_submit", "sid": "s1", "ts": 1.0, "turn_id": "t1"}, + {"event": "tool", "sid": "s1", "ts": 2.0, "tool": "Read", "ms": 10}, + ] + _, sessions = bdd.build_daily_and_sessions(events) + sess = sessions[0] + self.assertEqual(sess["timeline"][0]["events"][0]["agent"], "main") + + +class BuildDispatchTests(unittest.TestCase): + """真实数据复现过（TC5 的 dashboard-data-top-slow 会话，sid 01a09ef5...）： + 修复前 build_dispatch() 把 agent_history 里所有 evidence 以 "dispatch" 或 + "inbox" 开头的条目一律计数，导致两个真实问题—— + 1) solo-developer 被上报回声（inbox@report:*）重复计入，真实派发只有 2 次， + 却显示成 5； + 2) team-lead 显示成一个"被 main 派发的子 agent"，但 team-lead 在 + core/agent_identity.py::normalize_role_name 里本来就和 main 归为一类 + （它是 CodeBuddy 原生 team 基础设施里 lead session 自己的 mailbox 名， + 不是真实存在的子 agent），面板标题明明叫"Main → 子 Agent 派发次数"， + 混进一个其实等价于 main 自己的假子 agent。""" + + def _real_tc5_agent_history(self): + # 逐条取自真实会话 01a09ef5-1c6b-7928-870e-a5056ff360e1 的 .state.json。 + return [ + {"ts": 1.0, "agent": "solo-developer", "evidence": "dispatch@Agent<-main"}, + {"ts": 2.0, "agent": "team-lead", "evidence": "dispatch@SendMessage<-solo-developer"}, + {"ts": 3.0, "agent": "solo-developer", "evidence": "inbox@report:solo-developer"}, + {"ts": 4.0, "agent": "solo-developer", "evidence": "inbox@dispatch:solo-developer"}, + {"ts": 5.0, "agent": "main", "evidence": "dispatch@SendMessage<-solo-developer"}, + {"ts": 6.0, "agent": "main", "evidence": "inbox@handoff:solo-developer->main"}, + {"ts": 7.0, "agent": "solo-developer", "evidence": "dispatch@SendMessage<-main"}, + {"ts": 8.0, "agent": "solo-developer", "evidence": "inbox@report:solo-developer"}, + ] + + def test_real_tc5_history_excludes_team_lead_and_report_echoes(self): + state = {"01a09ef5": {"agent_history": self._real_tc5_agent_history()}} + rows = bdd.build_dispatch(state) + # team-lead（main 的别名）完全不出现；solo-developer 只数真正代表新派发的 + # 3 条证据（2 条工具触发的 dispatch@ + 1 条 inbox 独立确认的 dispatch:）， + # 2 条上报回声（inbox@report:*）不计入。 + self.assertEqual(rows, [{"agent": "solo-developer", "count": 3}]) + + def test_inbox_dispatch_evidence_alone_is_still_counted(self): + """inbox 扫描独立确认的 "main 派给了谁"（inbox@dispatch:*）是原生 team + 基础设施里唯一能感知到、但没有经过 Agent/Task 工具调用拦截到的派发方式 + （例如 team-lead 用 mailbox 直接投递任务），必须保留，不能因为过滤 + report/handoff 回声就连这类真实派发信号也一起丢掉。""" + state = {"s1": {"agent_history": [ + {"ts": 1.0, "agent": "architect", "evidence": "inbox@dispatch:architect"}, + ]}} + rows = bdd.build_dispatch(state) + self.assertEqual(rows, [{"agent": "architect", "count": 1}]) + + def test_report_and_handoff_evidence_alone_are_not_dispatches(self): + state = {"s1": {"agent_history": [ + {"ts": 1.0, "agent": "developer", "evidence": "inbox@report:developer"}, + {"ts": 2.0, "agent": "test-engineer", "evidence": "inbox@handoff:developer->test-engineer"}, + ]}} + rows = bdd.build_dispatch(state) + self.assertEqual(rows, []) + + def test_team_lead_is_excluded_even_without_report_noise(self): + state = {"s1": {"agent_history": [ + {"ts": 1.0, "agent": "team-lead", "evidence": "dispatch@SendMessage<-main"}, + ]}} + rows = bdd.build_dispatch(state) + self.assertEqual(rows, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/.codebuddy/skills/agent-observability/tests/test_cls_sink.py b/.codebuddy/skills/agent-observability/tests/test_cls_sink.py new file mode 100644 index 0000000..2da9d6a --- /dev/null +++ b/.codebuddy/skills/agent-observability/tests/test_cls_sink.py @@ -0,0 +1,184 @@ +from __future__ import annotations + +import json +import os +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "scripts" +if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) + +from core import cls_sink, emitter # type: ignore + + +class CLSSinkTests(unittest.TestCase): + def _write_cls_env(self, root: Path, *extra_lines: str) -> None: + lines = [ + "CLS_TOPIC_ID=test-topic-id", + "CLS_ENDPOINT=cls.internal.tencentcloudapi.com", + "CLS_SERVICE_NAME=agent-observability", + "CLS_UPLOAD_TIMEOUT_SECONDS=20", + *extra_lines, + ] + (root / ".env").write_text("\n".join(lines) + "\n", encoding="utf-8") + + def test_load_config_supports_cls_and_tc_credentials(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + self._write_cls_env( + root, + "TC_SECRET_ID=test-secret-id", + "TC_SECRET_KEY=test-secret-key", + ) + with mock.patch.dict(os.environ, {}, clear=True): + config = cls_sink.load_config(td) + + self.assertTrue(config.enabled) + self.assertEqual(config.endpoint, "cls.internal.tencentcloudapi.com") + self.assertEqual(config.topic_id, "test-topic-id") + self.assertEqual(config.secret_id, "test-secret-id") + self.assertEqual(config.secret_key, "test-secret-key") + self.assertEqual(config.secret_token, "") + self.assertEqual(config.service_name, "agent-observability") + self.assertEqual(config.timeout_seconds, 20) + self.assertTrue(config.ready) + + def test_load_config_supports_tencentcloud_scoped_credentials(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + self._write_cls_env(root) + with mock.patch.dict( + os.environ, + { + "TENCENTCLOUD_SECRET_ID_438167613": "scoped-secret-id", + "TENCENTCLOUD_SECRET_KEY_438167613": "scoped-secret-key", + }, + clear=True, + ): + config = cls_sink.load_config(td) + + self.assertEqual(config.secret_id, "scoped-secret-id") + self.assertEqual(config.secret_key, "scoped-secret-key") + self.assertEqual(config.timeout_seconds, 20) + self.assertTrue(config.ready) + + def test_mirror_record_invokes_node_uploader(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + self._write_cls_env( + root, + "CLS_SECRET_ID=test-secret-id", + "CLS_SECRET_KEY=test-secret-key", + ) + with mock.patch.object(cls_sink, "_put_logs_via_api", return_value=(False, "python-failed")), \ + mock.patch.object(cls_sink.subprocess, "run", return_value=mock.Mock(returncode=0, stdout="", stderr="")) as run, \ + mock.patch.dict(os.environ, {}, clear=True): + ok = cls_sink.mirror_record({"event": "tool", "sid": "s-1"}, cwd=td) + + self.assertTrue(ok) + run.assert_called_once() + payload = json.loads(run.call_args.kwargs["input"]) + self.assertEqual(payload["records"][0]["event"], "tool") + self.assertEqual(payload["topicId"], "test-topic-id") + self.assertEqual(payload["secretToken"], "") + + def test_load_config_requires_credentials_from_env(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + self._write_cls_env(root) + with mock.patch.dict(os.environ, {}, clear=True): + config = cls_sink.load_config(td) + + self.assertEqual(config.secret_id, "") + self.assertEqual(config.secret_key, "") + self.assertEqual(config.secret_token, "") + self.assertEqual(config.timeout_seconds, 20) + self.assertFalse(config.ready) + + def test_load_config_reads_secret_token_aliases(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + self._write_cls_env( + root, + "CLS_SECRET_ID=test-secret-id", + "CLS_SECRET_KEY=test-secret-key", + "TC_SESSION_TOKEN=test-session-token", + ) + with mock.patch.dict(os.environ, {}, clear=True): + config = cls_sink.load_config(td) + + self.assertEqual(config.secret_token, "test-session-token") + + def test_load_config_supports_timeout_override(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + self._write_cls_env( + root, + "CLS_SECRET_ID=test-secret-id", + "CLS_SECRET_KEY=test-secret-key", + "CLS_UPLOAD_TIMEOUT_SECONDS=45", + ) + with mock.patch.dict(os.environ, {}, clear=True): + config = cls_sink.load_config(td) + + self.assertEqual(config.timeout_seconds, 45) + + def test_mirror_record_writes_debug_when_uploader_fails(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + debug_path = root / "cls-push-debug.ndjson" + self._write_cls_env( + root, + "CLS_SECRET_ID=test-secret-id", + "CLS_SECRET_KEY=test-secret-key", + ) + with mock.patch.object(cls_sink, "debug_log_path", return_value=debug_path), \ + mock.patch.object(cls_sink, "_put_logs_via_api", return_value=(False, "python-failed")), \ + mock.patch.object(cls_sink.subprocess, "run", return_value=mock.Mock(returncode=1, stdout="", stderr="boom")), \ + mock.patch.dict(os.environ, {}, clear=True): + ok = cls_sink.mirror_record({"event": "tool", "sid": "s-2"}, cwd=td) + + entries = [json.loads(line) for line in debug_path.read_text("utf-8").splitlines() if line.strip()] + + self.assertFalse(ok) + self.assertEqual(entries[-1]["stage"], "uploader_failed") + self.assertEqual(entries[-1]["stderr"], "boom") + self.assertEqual(entries[-1]["timeout_seconds"], 20) + + def test_mirror_record_writes_success_debug_when_uploader_succeeds(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + debug_path = root / "cls-push-debug.ndjson" + self._write_cls_env( + root, + "CLS_SECRET_ID=test-secret-id", + "CLS_SECRET_KEY=test-secret-key", + ) + with mock.patch.object(cls_sink, "debug_log_path", return_value=debug_path), \ + mock.patch.object(cls_sink, "_put_logs_via_api", return_value=(True, "ok")), \ + mock.patch.dict(os.environ, {}, clear=True): + ok = cls_sink.mirror_record({"event": "tool", "sid": "s-3"}, cwd=td) + + entries = [json.loads(line) for line in debug_path.read_text("utf-8").splitlines() if line.strip()] + + self.assertTrue(ok) + self.assertEqual(entries[0]["stage"], "uploader_start") + self.assertEqual(entries[-1]["stage"], "uploader_ok") + self.assertEqual(entries[-1]["method"], "python_api_v3") + + def test_emit_keeps_local_log_when_cls_push_raises(self): + with tempfile.TemporaryDirectory() as td: + log_path = Path(td) / "metrics.ndjson" + with mock.patch.object(emitter.cls_sink, "mirror_record", side_effect=RuntimeError("boom")): + emitter.emit(log_path, {"event": "start", "sid": "s-emit"}) + + records = [json.loads(line) for line in log_path.read_text("utf-8").splitlines() if line.strip()] + + self.assertEqual(len(records), 1) + self.assertEqual(records[0]["event"], "start") + self.assertEqual(records[0]["sid"], "s-emit") diff --git a/.codebuddy/skills/agent-observability/tests/test_collector.py b/.codebuddy/skills/agent-observability/tests/test_collector.py new file mode 100644 index 0000000..05547f0 --- /dev/null +++ b/.codebuddy/skills/agent-observability/tests/test_collector.py @@ -0,0 +1,715 @@ +from __future__ import annotations + +import json +import tempfile +import time +import unittest +from pathlib import Path +import sys + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "scripts" +if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) + +from core import agent_identity, agentlens, collector, emitter, runtime, state as st # type: ignore + + +class CollectorTests(unittest.TestCase): + def test_cost_of_uses_known_model_price_table(self): + tokens = {"input": 1000, "output": 200, "cache_read": 0, "total": 1200} + cost = emitter.cost_of(tokens, "gpt-4o") + self.assertIsNotNone(cost) + self.assertGreaterEqual(cost, 0.0) + + def test_cost_of_returns_none_for_unknown_model(self): + tokens = {"input": 1000, "output": 200, "cache_read": 0, "total": 1200} + self.assertIsNone(emitter.cost_of(tokens, "unknown-model")) + + def test_emit_adds_codebuddy_cli_data_source(self): + with tempfile.TemporaryDirectory() as td: + log_path = Path(td) / "metrics.ndjson" + + emitter.emit(log_path, {"event": "start", "sid": "s-data-source"}) + + record = json.loads(log_path.read_text(encoding="utf-8").strip()) + self.assertEqual(record["data_source"], "codebuddy-cli") + + def test_emit_preserves_explicit_data_source(self): + with tempfile.TemporaryDirectory() as td: + log_path = Path(td) / "metrics.ndjson" + + emitter.emit(log_path, {"event": "start", "sid": "s-data-source", "data_source": "manual"}) + + record = json.loads(log_path.read_text(encoding="utf-8").strip()) + self.assertEqual(record["data_source"], "manual") + + def test_emit_session_duration_uses_first_log_ts_when_state_was_recreated(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + log_path = root / "metrics.ndjson" + state_path = root / ".state.json" + log_path.write_text( + json.dumps({"event": "start", "sid": "s-duration", "ts": 100.0}) + "\n", + encoding="utf-8", + ) + state_path.write_text( + json.dumps({"s-duration": {"started_at": 190.0}}, ensure_ascii=False), + encoding="utf-8", + ) + + emitter.emit(log_path, {"event": "usage", "sid": "s-duration", "ts": 200.0, "tokens": {"input": 1, "output": 2}}) + + records = [json.loads(line) for line in log_path.read_text(encoding="utf-8").splitlines()] + self.assertEqual(records[-1]["session_duration_sec"], 100.0) + + def test_emit_session_duration_starts_at_zero_without_prior_log_or_state(self): + with tempfile.TemporaryDirectory() as td: + log_path = Path(td) / "metrics.ndjson" + + emitter.emit(log_path, {"event": "start", "sid": "s-new", "ts": 100.0}) + + record = json.loads(log_path.read_text(encoding="utf-8").strip()) + self.assertEqual(record["session_duration_sec"], 0.0) + + def test_parse_transcript_tail_extracts_tokens(self): + transcript = ROOT / "tests" / "fixtures" / "transcript.jsonl" + out = collector.parse_transcript_tail(str(transcript), max_lines=20) + self.assertIsInstance(out, dict) + self.assertIsInstance(out.get("tokens"), dict) + self.assertEqual(out["tokens"].get("input_tokens"), 2240) + self.assertEqual(out["tokens"].get("output_tokens"), 120) + usage_records = out.get("usage_records") or [] + tool_records = out.get("tool_records") or [] + self.assertEqual(len(usage_records), 3) + self.assertEqual(usage_records[0]["tokens"].get("input_tokens"), 1820) + self.assertEqual(usage_records[-1]["tokens"].get("input_tokens"), 2240) + self.assertEqual(usage_records[0].get("message_id"), "msg-001") + self.assertEqual(usage_records[-1].get("message_id"), "msg-003") + self.assertLess(usage_records[0]["offset"], usage_records[-1]["offset"]) + self.assertEqual(len(tool_records), 8) + self.assertEqual(tool_records[0]["tool"], "read_file") + self.assertEqual(tool_records[0]["message_id"], "msg-001") + self.assertEqual(tool_records[0]["next_message_id"], "msg-002") + self.assertEqual(tool_records[-1]["tool"], "Bash") + self.assertEqual(tool_records[-1]["message_id"], "msg-002") + self.assertEqual(tool_records[-1]["next_message_id"], "msg-003") + + def test_realtime_usage_is_request_level_and_dedupes_across_phase(self): + transcript = ROOT / "tests" / "fixtures" / "transcript.jsonl" + with tempfile.TemporaryDirectory() as td: + state_path = Path(td) / ".state.json" + sid = "s-transcript" + + scan = collector.collect_transcript_entries( + state_path=state_path, + sid=sid, + transcript_path=str(transcript), + max_lines=20, + ) + entries = scan.get("entries") or [] + + self.assertEqual(len(entries), 3) + self.assertEqual(entries[0]["tokens"]["input"], 1820) + self.assertGreater(entries[0]["offset"], 0) + self.assertEqual(entries[0]["message_id"], "msg-001") + self.assertEqual(entries[1]["tokens"]["input"], 2100) + self.assertEqual(entries[1]["tokens"]["total"], 2158) + self.assertEqual(len(scan.get("tool_records") or []), 8) + + usage_key = collector.build_transcript_event_key( + kind="usage", + tool="Read", + entry=entries[0], + ) + stop_key = collector.build_transcript_event_key( + kind="stop", + tool="__stop__", + entry=entries[0], + ) + other_tool_key = collector.build_transcript_event_key( + kind="usage", + tool="Bash", + entry=entries[0], + ) + + self.assertEqual(usage_key, stop_key) + self.assertEqual(usage_key, other_tool_key) + self.assertTrue(st.claim_transcript_event(state_path, sid, usage_key, str(transcript))) + self.assertFalse(st.claim_transcript_event(state_path, sid, stop_key, str(transcript))) + + def test_find_current_tool_message_id_prefers_transcript_tool_records(self): + transcript = ROOT / "tests" / "fixtures" / "transcript.jsonl" + with tempfile.TemporaryDirectory() as td: + state_path = Path(td) / ".state.json" + sid = "s-tool-mid" + + read_mid = collector.find_current_tool_message_id( + state_path, + sid, + str(transcript), + "Read", + ) + bash_mid = collector.find_current_tool_message_id( + state_path, + sid, + str(transcript), + "Bash", + ) + + self.assertEqual(read_mid, "msg-002") + self.assertEqual(bash_mid, "msg-003") + + def test_resolve_subagent_transcript_alias_and_tool_context(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + sid = "session-123" + main = root / f"{sid}.jsonl" + bundle = root / sid + subagents = bundle / "subagents" + subagents.mkdir(parents=True) + alias_session_id = "subagent-session-1" + subagent = subagents / "agent-a.jsonl" + main.write_text("", encoding="utf-8") + subagent.write_text( + "\n".join( + [ + json.dumps( + { + "type": "message", + "sessionId": alias_session_id, + "providerData": {"agent": "Explore"}, + } + ), + json.dumps( + { + "type": "function_call", + "sessionId": alias_session_id, + "name": "Grep", + "callId": "call-1", + "providerData": { + "agent": "Explore", + "messageId": "msg-sub-001", + }, + } + ), + ] + ) + + "\n", + encoding="utf-8", + ) + + aliased_path = str(root / f"{alias_session_id}.jsonl") + resolved = collector.resolve_transcript_path_alias(sid, aliased_path) + self.assertEqual(resolved, str(subagent.resolve())) + + state_path = root / ".state.json" + context = collector.find_current_tool_context( + state_path=state_path, + sid=sid, + transcript_path=aliased_path, + tool_name="Grep", + ) + self.assertEqual( + context, + { + "message_id": "msg-sub-001", + "transcript_path": str(subagent.resolve()), + "agent": "explore", + "tool_details": {"call_id": "call-1"}, + }, + ) + + def test_find_current_tool_context_prefers_nearest_timestamp(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + transcript = root / "session.jsonl" + transcript.write_text( + "\n".join( + [ + json.dumps( + { + "timestamp": 1000, + "type": "function_call", + "name": "Bash", + "callId": "call-1", + "providerData": {"messageId": "msg-early"}, + } + ), + json.dumps( + { + "timestamp": 5000, + "type": "function_call", + "name": "Bash", + "callId": "call-2", + "providerData": {"messageId": "msg-late"}, + } + ), + ] + ) + + "\n", + encoding="utf-8", + ) + state_path = root / ".state.json" + + early = collector.find_current_tool_context( + state_path=state_path, + sid="s-nearest", + transcript_path=str(transcript), + tool_name="Bash", + event_ts=1.2, + ) + late = collector.find_current_tool_context( + state_path=state_path, + sid="s-nearest", + transcript_path=str(transcript), + tool_name="Bash", + event_ts=4.9, + ) + + self.assertEqual(early["message_id"], "msg-early") + self.assertEqual(late["message_id"], "msg-late") + + def test_find_current_tool_context_claims_distinct_call_ids_with_same_timestamp(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + transcript = root / "session.jsonl" + transcript.write_text( + "\n".join( + [ + json.dumps( + { + "timestamp": 5000, + "type": "function_call", + "name": "Bash", + "callId": "call-1", + "arguments": "{\"command\":\"echo one\"}", + "providerData": {"messageId": "msg-shared"}, + } + ), + json.dumps( + { + "timestamp": 5000, + "type": "function_call", + "name": "Bash", + "callId": "call-2", + "arguments": "{\"command\":\"echo two\"}", + "providerData": {"messageId": "msg-shared"}, + } + ), + json.dumps( + { + "timestamp": 5000, + "type": "function_call_result", + "name": "Bash", + "callId": "call-1", + "providerData": { + "messageId": "msg-shared", + "toolResult": {"content": "result one"}, + }, + } + ), + json.dumps( + { + "timestamp": 5000, + "type": "function_call_result", + "name": "Bash", + "callId": "call-2", + "providerData": { + "messageId": "msg-shared", + "toolResult": {"content": "result two"}, + }, + } + ), + ] + ) + + "\n", + encoding="utf-8", + ) + state_path = root / ".state.json" + + first = collector.find_current_tool_context( + state_path=state_path, + sid="s-claim-shared", + transcript_path=str(transcript), + tool_name="Bash", + event_ts=4.9, + claim=True, + ) + second = collector.find_current_tool_context( + state_path=state_path, + sid="s-claim-shared", + transcript_path=str(transcript), + tool_name="Bash", + event_ts=4.9, + claim=True, + ) + third = collector.find_current_tool_context( + state_path=state_path, + sid="s-claim-shared", + transcript_path=str(transcript), + tool_name="Bash", + event_ts=4.9, + claim=True, + ) + + self.assertEqual(first["message_id"], "msg-shared") + self.assertEqual(second["message_id"], "msg-shared") + self.assertNotEqual(first["tool_details"]["call_id"], second["tool_details"]["call_id"]) + self.assertIn("arguments", first["tool_details"]) + self.assertIn("result_content", first["tool_details"]) + self.assertEqual(third, {"duplicate": True}) + + def test_find_fallback_usage_event_prefers_latest_matching_usage(self): + tool_event = { + "tool": "Bash", + "agent": "main", + "transcript_path": "/tmp/demo.jsonl", + } + usage_events = [ + { + "agent": "main", + "tool": "Read", + "transcript_path": "/tmp/demo.jsonl", + "message_id": "msg-old", + "source_offset": 100, + }, + { + "agent": "main", + "tool": "Bash", + "transcript_path": "/tmp/demo.jsonl", + "message_id": "msg-current", + "source_offset": 200, + }, + ] + + matched = collector.find_fallback_usage_event(tool_event, usage_events) + self.assertIsNotNone(matched) + self.assertEqual(matched["message_id"], "msg-current") + + def test_find_current_tool_context_prefers_explicit_call_id_over_nearest_timestamp(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + transcript = root / "session.jsonl" + transcript.write_text( + "\n".join( + [ + json.dumps( + { + "timestamp": 1000, + "type": "function_call", + "name": "Bash", + "callId": "call-early", + "providerData": {"messageId": "msg-early"}, + } + ), + json.dumps( + { + "timestamp": 1001, + "type": "function_call_result", + "name": "Bash", + "callId": "call-early", + "providerData": { + "messageId": "msg-early", + "toolResult": {"content": "early result"}, + }, + } + ), + json.dumps( + { + "timestamp": 5000, + "type": "function_call", + "name": "Bash", + "callId": "call-late", + "providerData": {"messageId": "msg-late"}, + } + ), + json.dumps( + { + "timestamp": 5001, + "type": "function_call_result", + "name": "Bash", + "callId": "call-late", + "providerData": { + "messageId": "msg-late", + "toolResult": {"content": "late result"}, + }, + } + ), + ] + ) + + "\n", + encoding="utf-8", + ) + state_path = root / ".state.json" + + context = collector.find_current_tool_context( + state_path=state_path, + sid="s-call-id-priority", + transcript_path=str(transcript), + tool_name="Bash", + call_id="call-early", + event_ts=4.9, + ) + + self.assertEqual(context["message_id"], "msg-early") + self.assertEqual(context["tool_details"]["call_id"], "call-early") + self.assertEqual(context["tool_details"]["result_content"], "early result") + + def test_find_current_tool_context_uses_next_message_id_when_available(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + transcript = root / "session.jsonl" + transcript.write_text( + "\n".join( + [ + json.dumps( + { + "timestamp": 1000, + "type": "function_call", + "name": "Read", + "callId": "call-1", + "providerData": {"messageId": "msg-tool"}, + } + ), + json.dumps( + { + "timestamp": 1001, + "type": "function_call_result", + "name": "Read", + "callId": "call-1", + "providerData": { + "messageId": "msg-tool", + "toolResult": {"content": "missing"}, + }, + } + ), + json.dumps( + { + "timestamp": 2000, + "type": "assistant", + "providerData": {"messageId": "msg-next"}, + "content": "next step", + } + ), + ] + ) + + "\n", + encoding="utf-8", + ) + state_path = root / ".state.json" + + context = collector.find_current_tool_context( + state_path=state_path, + sid="s-next-mid", + transcript_path=str(transcript), + tool_name="Read", + call_id="call-1", + ) + + self.assertEqual(context["message_id"], "msg-next") + self.assertEqual(context["tool_details"]["original_message_id"], "msg-tool") + self.assertEqual(context["tool_details"]["next_message_id"], "msg-next") + self.assertTrue(context["tool_details"]["message_id_reassigned"]) + + def test_flush_pending_tool_events_emits_after_transcript_catches_up(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + state_path = root / ".state.json" + log_path = root / "metrics.ndjson" + transcript = root / "session.jsonl" + transcript.write_text("", encoding="utf-8") + sid = "s-pending" + + tool_event = { + "event": "tool", + "sid": sid, + "agent": "main", + "tool": "Bash", + "ms": 42, + "transcript_path": str(transcript), + "turn_id": "turn-1", + "ts": 1.2, + "skill": [], + "rule": [], + "cwd": str(ROOT), + } + st.append_pending_tool_emit(state_path, sid, tool_event) + + transcript.write_text( + json.dumps( + { + "timestamp": 1300, + "type": "function_call", + "name": "Bash", + "callId": "call-1", + "providerData": {"messageId": "msg-later"}, + } + ) + + "\n", + encoding="utf-8", + ) + + emitted_calls: list[dict[str, object]] = [] + original_emit_post_step = agentlens.emit_post_step + try: + agentlens.emit_post_step = lambda **kwargs: emitted_calls.append(kwargs) + runtime.flush_pending_tool_events( + state_path=state_path, + log_path=log_path, + sid=sid, + ) + finally: + agentlens.emit_post_step = original_emit_post_step + + self.assertEqual(st.get_pending_tool_emits(state_path, sid), []) + lines = [json.loads(line) for line in log_path.read_text(encoding="utf-8").splitlines() if line.strip()] + self.assertEqual(len(lines), 1) + self.assertEqual(lines[0]["message_id"], "msg-later") + self.assertEqual(len(emitted_calls), 1) + self.assertEqual(emitted_calls[0]["tool_event"]["message_id"], "msg-later") + + def test_related_transcript_paths_include_subagents(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + sid = "session-123" + main = root / f"{sid}.jsonl" + bundle = root / sid + subagents = bundle / "subagents" + subagents.mkdir(parents=True) + sub_a = subagents / "agent-a.jsonl" + sub_b = subagents / "agent-b.jsonl" + for path in (main, sub_a, sub_b): + path.write_text("", encoding="utf-8") + + paths = collector.related_transcript_paths(sid, str(main)) + + self.assertEqual( + paths, + [ + str(main.resolve()), + str(sub_a.resolve()), + str(sub_b.resolve()), + ], + ) + + def test_subagent_transcript_maps_to_role_name(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + subagent = root / "subagents" / "agent-27e41af0.jsonl" + subagent.parent.mkdir(parents=True) + subagent.write_text( + '{"content":[{"text":""}]}\n', + encoding="utf-8", + ) + + self.assertEqual( + agent_identity.agent_for_transcript_path(str(subagent), "main"), + "knowledge-engineer", + ) + + def test_subagent_transcript_prefers_assignment_role_over_general_provider_agent(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + subagent = root / "subagents" / "agent-165e361f.jsonl" + subagent.parent.mkdir(parents=True) + subagent.write_text( + '{"type":"message","providerData":{"agent":"general-purpose"},"content":[{"text":"\\n待命。\\n你在本 devflow 中的角色：developer（开发角色）"}]}\n', + encoding="utf-8", + ) + + self.assertEqual( + agent_identity.agent_for_transcript_path(str(subagent), "main"), + "developer", + ) + + def test_inbox_standby_assignment_does_not_switch_current_agent(self): + messages = [ + { + "mailbox_name": "architect", + "mailbox_role": "architect", + "from_role": "main", + "summary": "Initial task assignment for architect", + "text": "待命，监听 main 唤醒。", + "payload": None, + "is_shutdown": False, + "is_standby": True, + } + ] + current, dispatched, meta = agent_identity.infer_identity_from_messages(messages) + self.assertIsNone(current) + self.assertIsNone(dispatched) + self.assertEqual(meta["messages_seen"], 1) + + def test_record_pre_post_duration(self): + with tempfile.TemporaryDirectory() as td: + pending = Path(td) / ".pending.json" + pre = {"session_id": "s1", "tool_name": "read_file"} + post = {"session_id": "s1", "tool_name": "read_file"} + collector.record_pre(pending, pre) + time.sleep(0.02) + ms = collector.record_post(pending, post) + self.assertIsInstance(ms, int) + self.assertGreaterEqual(ms, 0) + + def test_record_tool_usage_counts_skill_and_rule(self): + with tempfile.TemporaryDirectory() as td: + state_path = Path(td) / ".state.json" + sid = "s-tool" + # 先确保 session 结构 + s = st.load_state(state_path) + st.ensure_session(s, sid) + st.save_state(state_path, s) + + data = { + "tool_name": "use_skill", + "tool_input": {"command": "pdf", "filePath": "README.md"}, + } + skills_meta = {"pdf": {"source": "user", "version": "1.0.0"}} + rules_meta = { + "security": {"source": "project", "alwaysApply": True, "enabled": True, "globs": []} + } + collector.record_tool_usage( + state_path=state_path, + sid=sid, + data=data, + skills_meta=skills_meta, + rules_meta=rules_meta, + active_agent="main", + collect_skills=True, + ) + + skills_usage, rules_usage = collector.get_session_usage(state_path, sid) + self.assertEqual(skills_usage.get("pdf", {}).get("count"), 1) + self.assertEqual(rules_usage.get("security", {}).get("count"), 1) + + def test_step_span_registry_is_idempotent(self): + with tempfile.TemporaryDirectory() as td: + state_path = Path(td) / ".state.json" + sid = "s-step" + carrier = {"traceparent": "00-" + ("1" * 32) + "-" + ("2" * 16) + "-01"} + st.begin_turn(state_path, sid, "turn-1", carrier) + + first = st.upsert_step_span( + state_path, + sid, + "main", + "msg-001", + {"traceparent": "00-" + ("1" * 32) + "-" + ("3" * 16) + "-01"}, + transcript_path="/tmp/demo.jsonl", + ) + second = st.upsert_step_span( + state_path, + sid, + "main", + "msg-001", + {"traceparent": "00-" + ("1" * 32) + "-" + ("4" * 16) + "-01"}, + transcript_path="/tmp/demo.jsonl", + ) + + self.assertEqual(first, second) + self.assertEqual(st.get_step_span_carrier(state_path, sid, "main", "msg-001"), first) + + +if __name__ == "__main__": + unittest.main() diff --git a/.codebuddy/skills/agent-observability/tests/test_devflow.py b/.codebuddy/skills/agent-observability/tests/test_devflow.py new file mode 100644 index 0000000..9ad0a7f --- /dev/null +++ b/.codebuddy/skills/agent-observability/tests/test_devflow.py @@ -0,0 +1,389 @@ +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path +import sys + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "scripts" +if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) + +from core import agent_identity, devflow, state as st # type: ignore + + +def _write_team_config(team_dir: Path, *, lead_session_id: str, member_cwd: str) -> None: + team_dir.mkdir(parents=True, exist_ok=True) + (team_dir / "config.json").write_text( + json.dumps({ + "leadSessionId": lead_session_id, + "createdAt": 1, + "members": [{"cwd": member_cwd}], + }), + encoding="utf-8", + ) + + +class TaskSlugFromTeamDirTests(unittest.TestCase): + def test_strips_fixed_prefix(self): + team_dir = Path("/tmp/.codebuddy/teams/multi-agents-devflow-my-task_20260911_1200") + self.assertEqual(devflow.task_slug_from_team_dir(team_dir), "my-task_20260911_1200") + + def test_returns_none_for_non_devflow_team_dir(self): + team_dir = Path("/tmp/.codebuddy/teams/some-other-team") + self.assertIsNone(devflow.task_slug_from_team_dir(team_dir)) + + +class StageSnapshotTests(unittest.TestCase): + def test_projects_expected_fields_and_ignores_non_dict_stage(self): + workflow_state = { + "current_stage": "TASK-03", + "size_class": "medium", + "run_mode": "auto", + "stages": { + "TASK-02": {"status": "completed", "executor": "architect", "retry_count": 0, "review_result": None}, + "CODE-REVIEW": {"status": "failed", "executor": "code-reviewer", "retry_count": 1, "review_result": "failed"}, + "garbage": "not-a-dict", + }, + } + snap = devflow.stage_snapshot(workflow_state) + self.assertEqual(snap["current_stage"], "TASK-03") + self.assertEqual(snap["size_class"], "medium") + self.assertNotIn("garbage", snap["stages"]) + self.assertEqual(snap["stages"]["CODE-REVIEW"]["retry_count"], 1) + self.assertEqual(snap["stages"]["CODE-REVIEW"]["review_result"], "failed") + + def test_handles_missing_or_malformed_input(self): + self.assertEqual(devflow.stage_snapshot(None), {}) + self.assertEqual(devflow.stage_snapshot({"stages": "not-a-dict"}), { + "current_stage": None, "size_class": None, "run_mode": None, + "schema_version": None, "execution_mode": None, "host_adapter": None, "run_id": None, + "stages": {}, + }) + + +class DiffStageChangesTests(unittest.TestCase): + def test_first_observation_produces_no_changes(self): + curr = devflow.stage_snapshot({"stages": {"TASK-02": {"status": "completed", "retry_count": 0}}}) + self.assertEqual(devflow.diff_stage_changes(None, curr), []) + + def test_detects_retry_count_increase_as_a_change(self): + prev = devflow.stage_snapshot({"stages": {"CODE-REVIEW": {"status": "in_progress", "retry_count": 0}}}) + curr = devflow.stage_snapshot({"stages": {"CODE-REVIEW": {"status": "failed", "retry_count": 1, "review_result": "failed"}}}) + changes = devflow.diff_stage_changes(prev, curr) + self.assertEqual(len(changes), 1) + self.assertEqual(changes[0]["stage"], "CODE-REVIEW") + self.assertEqual(changes[0]["retry_count"], 1) + self.assertEqual(changes[0]["review_result"], "failed") + + def test_unchanged_stage_produces_no_entry(self): + snap = devflow.stage_snapshot({"stages": {"TASK-02": {"status": "completed", "retry_count": 0}}}) + self.assertEqual(devflow.diff_stage_changes(snap, snap), []) + + +class PtIdFromTranscriptPathTests(unittest.TestCase): + def test_extracts_pt_track_from_subagent_transcript_name(self): + path = "/tmp/session-123/subagents/sub-developer-PT-01.jsonl" + self.assertEqual(agent_identity.pt_id_from_transcript_path(path), "PT-01") + + def test_returns_none_for_non_subagent_or_non_pt_transcript(self): + self.assertIsNone(agent_identity.pt_id_from_transcript_path("/tmp/session-123.jsonl")) + self.assertIsNone(agent_identity.pt_id_from_transcript_path("/tmp/session-123/subagents/architect.jsonl")) + + +class ResolveAndDiffTests(unittest.TestCase): + def test_returns_none_when_no_devflow_team_found(self): + with tempfile.TemporaryDirectory() as td: + state_path = Path(td) / ".state.json" + result = devflow.resolve_and_diff(state_path, "sid-1", "/tmp/some-project") + self.assertIsNone(result) + state = st.load_state(state_path) + self.assertIsNone(state["sid-1"]["_devflow"]["context"]) + + def test_late_appearing_devflow_run_is_discovered_on_a_later_call(self): + """真实 bug 回归测试：同一个长生命周期 sid，会话开始时探测不到 devflow + （此时既没有 team 目录也没有 artifacts/），之后才真正跑起 devflow + （比如通过 Agent 工具后台 spawn，而不是从一开始就是独立 team 成员 sid）。 + 早期的"没找到"不能被永久缓存，必须在 artifacts/ 出现之后的下一次调用里发现它。""" + with tempfile.TemporaryDirectory() as td: + project_dir = Path(td) / "project" + project_dir.mkdir() + state_path = project_dir / ".codebuddy" / "skills" / "agent-observability" / "logs" / ".state.json" + sid = "sid-long-lived" + + # 会话早期：还没有任何 devflow 痕迹。 + first = devflow.resolve_and_diff(state_path, sid, str(project_dir)) + self.assertIsNone(first) + + # 同一个 sid，会话中途才出现 devflow 产物（没有 team 目录，走 artifacts 扫描兜底）。 + artifacts_dir = project_dir / "artifacts" / "late-appearing-task_20260914_1200" + artifacts_dir.mkdir(parents=True) + (artifacts_dir / "workflow-state.json").write_text(json.dumps({ + "version": "1.3", "current_stage": "SOLO", "size_class": "small", + "stages": {"SOLO": {"status": "completed", "executor": "solo-developer", "retry_count": 0}}, + }), encoding="utf-8") + + second = devflow.resolve_and_diff(state_path, sid, str(project_dir)) + self.assertIsNotNone(second) + self.assertEqual(second["task_slug"], "late-appearing-task_20260914_1200") + + def test_end_to_end_with_fake_team_and_workflow_state(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + project_dir = root / "project" + project_dir.mkdir() + config_home = root / "codebuddy-home" + teams_root = config_home / "teams" + team_dir = teams_root / "multi-agents-devflow-fix-token-bypass_20260911_0900" + _write_team_config(team_dir, lead_session_id="sid-42", member_cwd=str(project_dir)) + + artifacts_dir = project_dir / "artifacts" / "fix-token-bypass_20260911_0900" + artifacts_dir.mkdir(parents=True) + workflow_state_path = artifacts_dir / "workflow-state.json" + workflow_state_path.write_text(json.dumps({ + "current_stage": "TASK-03", + "size_class": "medium", + "stages": { + "TASK-02": {"status": "completed", "executor": "architect", "retry_count": 0}, + "TASK-03": {"status": "in_progress", "executor": "developer", "retry_count": 0}, + }, + }), encoding="utf-8") + + import os + old_env = os.environ.get("CODEBUDDY_CONFIG_DIR") + os.environ["CODEBUDDY_CONFIG_DIR"] = str(config_home) + try: + state_path = project_dir / ".codebuddy" / "skills" / "agent-observability" / "logs" / ".state.json" + sid = "sid-42" + + # 第一次调用：发现 devflow 上下文，但因为是首次观测不产出 changes。 + first = devflow.resolve_and_diff(state_path, sid, str(project_dir)) + self.assertIsNotNone(first) + self.assertEqual(first["task_slug"], "fix-token-bypass_20260911_0900") + self.assertEqual(first["current_stage"], "TASK-03") + self.assertEqual(first["changes"], []) + + # workflow-state.json 更新：TASK-03 打回重试。 + workflow_state_path.write_text(json.dumps({ + "current_stage": "TASK-03", + "size_class": "medium", + "stages": { + "TASK-02": {"status": "completed", "executor": "architect", "retry_count": 0}, + "TASK-03": {"status": "failed", "executor": "developer", "retry_count": 1}, + }, + }), encoding="utf-8") + + second = devflow.resolve_and_diff(state_path, sid, str(project_dir)) + self.assertEqual(len(second["changes"]), 1) + self.assertEqual(second["changes"][0]["stage"], "TASK-03") + self.assertEqual(second["changes"][0]["retry_count"], 1) + self.assertEqual(second["changes"][0]["status"], "failed") + + # 没有变化时，第三次调用应该不再产出 changes。 + third = devflow.resolve_and_diff(state_path, sid, str(project_dir)) + self.assertEqual(third["changes"], []) + finally: + if old_env is None: + os.environ.pop("CODEBUDDY_CONFIG_DIR", None) + else: + os.environ["CODEBUDDY_CONFIG_DIR"] = old_env + + +class PortableSchemaStageSnapshotTests(unittest.TestCase): + """Portable（v2.0）用 `executor_role` 而不是 `executor`，且没有 `review_result`。""" + + def test_reads_executor_role_and_top_level_execution_context(self): + workflow_state = { + "version": "2.0", + "current_stage": "IMPLEMENT", + "size_class": "medium", + "execution_mode": "isolated", + "host_adapter": "codebuddy", + "run_id": "run-abc", + "stages": { + "DESIGN": {"status": "completed", "executor_role": "devflow-architect", "retry_count": 0}, + "IMPLEMENT": {"status": "failed", "executor_role": "devflow-developer", "retry_count": 1}, + }, + } + snap = devflow.stage_snapshot(workflow_state) + self.assertEqual(snap["schema_version"], "2.0") + self.assertEqual(snap["execution_mode"], "isolated") + self.assertEqual(snap["stages"]["DESIGN"]["executor"], "devflow-architect") + self.assertEqual(snap["stages"]["IMPLEMENT"]["retry_count"], 1) + self.assertIsNone(snap["stages"]["IMPLEMENT"]["review_result"]) + + def test_diff_detects_retry_on_portable_shape_same_as_classic(self): + prev = devflow.stage_snapshot({ + "version": "2.0", + "stages": {"IMPLEMENT": {"status": "in_progress", "executor_role": "devflow-developer", "retry_count": 0}}, + }) + curr = devflow.stage_snapshot({ + "version": "2.0", + "stages": {"IMPLEMENT": {"status": "failed", "executor_role": "devflow-developer", "retry_count": 1}}, + }) + changes = devflow.diff_stage_changes(prev, curr) + self.assertEqual(len(changes), 1) + self.assertEqual(changes[0]["stage"], "IMPLEMENT") + self.assertEqual(changes[0]["retry_count"], 1) + self.assertEqual(changes[0]["executor"], "devflow-developer") + + +class ArtifactsScanFallbackTests(unittest.TestCase): + """`topology: spawn` 宿主没有 team 目录，靠扫 artifacts/ 兜底发现 task_slug。""" + + def test_no_artifacts_dir_returns_none(self): + with tempfile.TemporaryDirectory() as td: + self.assertIsNone(devflow._scan_artifacts_for_active_run(td)) + + def test_ignores_non_devflow_json_and_picks_in_progress_run(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + artifacts = root / "artifacts" + + # 不是 devflow 产物的 JSON（没有 stages 字段），不能被误当成一次运行。 + noise_dir = artifacts / "not-a-devflow-run" + noise_dir.mkdir(parents=True) + (noise_dir / "workflow-state.json").write_text(json.dumps({"hello": "world"}), encoding="utf-8") + + done_dir = artifacts / "finished-task_20260910_0900" + done_dir.mkdir(parents=True) + (done_dir / "workflow-state.json").write_text(json.dumps({ + "status": "completed", "stages": {"DESIGN": {"status": "completed"}}, + }), encoding="utf-8") + + active_dir = artifacts / "active-task_20260911_1000" + active_dir.mkdir(parents=True) + (active_dir / "workflow-state.json").write_text(json.dumps({ + "status": "in_progress", "stages": {"IMPLEMENT": {"status": "in_progress"}}, + }), encoding="utf-8") + + result = devflow._scan_artifacts_for_active_run(str(root)) + self.assertIsNotNone(result) + self.assertEqual(result["task_slug"], "active-task_20260911_1000") + self.assertIsNone(result["team_dir"]) + + def test_finished_detection_works_for_classic_schema_without_top_level_status(self): + """Classic（v1.3）没有顶层 status 字段，"跑完没跑完"只能看 last_event。""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) + artifacts = root / "artifacts" + + done_dir = artifacts / "classic-done_20260910_0900" + done_dir.mkdir(parents=True) + (done_dir / "workflow-state.json").write_text(json.dumps({ + "version": "1.3", "last_event": "workflow_completed", + "stages": {"SOLO": {"status": "completed"}}, + }), encoding="utf-8") + + active_dir = artifacts / "classic-active_20260911_1000" + active_dir.mkdir(parents=True) + (active_dir / "workflow-state.json").write_text(json.dumps({ + "version": "1.3", "last_event": "TASK-02_completed", + "stages": {"TASK-02": {"status": "completed"}, "TASK-03": {"status": "in_progress"}}, + }), encoding="utf-8") + + result = devflow._scan_artifacts_for_active_run(str(root)) + self.assertIsNotNone(result) + self.assertEqual(result["task_slug"], "classic-active_20260911_1000") + + def test_resolve_and_diff_switches_task_slug_when_a_newer_run_appears(self): + """真实 bug 回归测试：同一个长生命周期 sid 先后归属两次不同的 devflow 运行 + （没有真正 team_create 时的降级场景——第二次 `/start-devflow` 复用了同一个 + session）。第一次探测缓存下的 task_slug 不能在第二次运行开始后继续沿用； + 必须切换到新的那个，而且两个 task_slug 各自的 diff 历史不能互相污染。""" + with tempfile.TemporaryDirectory() as td: + project_dir = Path(td) / "project" + state_path = project_dir / ".codebuddy" / "skills" / "agent-observability" / "logs" / ".state.json" + sid = "sid-reused-across-two-runs" + + first_dir = project_dir / "artifacts" / "first-task_20260914_0900" + first_dir.mkdir(parents=True) + (first_dir / "workflow-state.json").write_text(json.dumps({ + "version": "1.3", "last_event": "workflow_completed", "current_stage": "SOLO", + "stages": {"SOLO": {"status": "completed", "retry_count": 0}}, + }), encoding="utf-8") + + first = devflow.resolve_and_diff(state_path, sid, str(project_dir)) + self.assertEqual(first["task_slug"], "first-task_20260914_0900") + + # 同一个 sid，第二次运行出现，且比第一次更新（mtime 更新）。 + import time + time.sleep(0.01) + second_dir = project_dir / "artifacts" / "second-task_20260914_1100" + second_dir.mkdir(parents=True) + (second_dir / "workflow-state.json").write_text(json.dumps({ + "version": "1.3", "last_event": "workflow_completed", "current_stage": "SOLO", + "stages": {"SOLO": {"status": "completed", "retry_count": 1}}, + }), encoding="utf-8") + + second = devflow.resolve_and_diff(state_path, sid, str(project_dir)) + self.assertEqual(second["task_slug"], "second-task_20260914_1100") + self.assertEqual(second["changes"], []) # 对 second 是首次观测，不产出变更 + + # 第一个 task_slug 的历史没有被污染：如果它重新变成"最新"（比如被再次修改）， + # 应该正确切回，并且不会把 second 的历史错当成 first 的基线。 + (first_dir / "workflow-state.json").write_text(json.dumps({ + "version": "1.3", "last_event": "workflow_completed", "current_stage": "SOLO", + "stages": {"SOLO": {"status": "completed", "retry_count": 1}}, + }), encoding="utf-8") + time.sleep(0.01) + first_dir.joinpath("workflow-state.json").touch() + + third = devflow.resolve_and_diff(state_path, sid, str(project_dir)) + self.assertEqual(third["task_slug"], "first-task_20260914_0900") + # first 上一次被观测到时 retry_count 还是 0，现在变成 1——应该被识别为变化， + # 而不是被 second 的快照历史污染成"首次观测"或者对不上的 diff。 + self.assertEqual(len(third["changes"]), 1) + self.assertEqual(third["changes"][0]["retry_count"], 1) + + def test_resolve_and_diff_end_to_end_via_artifacts_scan_no_team(self): + with tempfile.TemporaryDirectory() as td: + project_dir = Path(td) / "project" + artifacts_dir = project_dir / "artifacts" / "portable-task_20260911_1100" + artifacts_dir.mkdir(parents=True) + state_file = artifacts_dir / "workflow-state.json" + state_file.write_text(json.dumps({ + "version": "2.0", + "status": "in_progress", + "current_stage": "REVIEW", + "execution_mode": "isolated", + "stages": {"REVIEW": {"status": "in_progress", "executor_role": "devflow-code-reviewer", "retry_count": 0}}, + }), encoding="utf-8") + + import os + old_env = os.environ.get("CODEBUDDY_CONFIG_DIR") + os.environ["CODEBUDDY_CONFIG_DIR"] = str(Path(td) / "empty-codebuddy-home") + try: + state_path = project_dir / ".codebuddy" / "skills" / "agent-observability" / "logs" / ".state.json" + sid = "sid-spawn-1" + + first = devflow.resolve_and_diff(state_path, sid, str(project_dir)) + self.assertIsNotNone(first) + self.assertEqual(first["task_slug"], "portable-task_20260911_1100") + self.assertEqual(first["schema_version"], "2.0") + self.assertEqual(first["execution_mode"], "isolated") + self.assertEqual(first["changes"], []) + + state_file.write_text(json.dumps({ + "version": "2.0", + "status": "in_progress", + "current_stage": "REVIEW", + "execution_mode": "isolated", + "stages": {"REVIEW": {"status": "failed", "executor_role": "devflow-code-reviewer", "retry_count": 1}}, + }), encoding="utf-8") + + second = devflow.resolve_and_diff(state_path, sid, str(project_dir)) + self.assertEqual(len(second["changes"]), 1) + self.assertEqual(second["changes"][0]["retry_count"], 1) + self.assertEqual(second["changes"][0]["status"], "failed") + finally: + if old_env is None: + os.environ.pop("CODEBUDDY_CONFIG_DIR", None) + else: + os.environ["CODEBUDDY_CONFIG_DIR"] = old_env + + +if __name__ == "__main__": + unittest.main() diff --git a/.codebuddy/skills/agent-observability/tests/test_pricing_overrides.py b/.codebuddy/skills/agent-observability/tests/test_pricing_overrides.py new file mode 100644 index 0000000..75615a3 --- /dev/null +++ b/.codebuddy/skills/agent-observability/tests/test_pricing_overrides.py @@ -0,0 +1,394 @@ +"""自定义模型定价覆盖 + 看板未定价模型提示。 + +覆盖文件是**人手写的常驻配置**，而且加载失败要静默降级——这类"错了也不许 +报错"的代码最容易在半年后被悄悄改坏（比如有人顺手把 try/except 去掉、把合并 +改成整 key 覆盖、或让降级路径开始 print）。这里把字段级合并、四种/五种降级 +路径、缓存语义、未定价口径、看板聚合与"不重算历史成本"逐条钉住。 + +两个容易踩的隔离坑，本文件统一在 setUp/tearDown 处理： +1. `load_prices()` 带 `lru_cache`，不 `clear_price_cache()` 用例之间会串味； +2. `AOBS_PRICING_OVERRIDES_PATH` 是进程级环境变量，不还原会污染同进程里 + 其它测试模块（尤其是会真的去读默认覆盖文件的 hook 链路）。 + +真实日志对账（hy4-preview-ioa / hy3-ioa 的计数）不写进单测：日志会持续增长， +写死数字会变成 flaky 用例。那部分以人工冒烟的方式在 TASK-04 报告里记录。 +""" +from __future__ import annotations + +import contextlib +import io +import json +import os +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "scripts" +if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) + +import build_dashboard_data as bdd # type: ignore +from core import emitter # type: ignore + +OVERRIDES_ENV = "AOBS_PRICING_OVERRIDES_PATH" + + +class _OverridesTestCase(unittest.TestCase): + """统一处理环境变量与 `load_prices()` 缓存的隔离。""" + + def setUp(self): + self._env_backup = os.environ.get(OVERRIDES_ENV) + os.environ.pop(OVERRIDES_ENV, None) + emitter.clear_price_cache() + self._tmp = tempfile.TemporaryDirectory() + self.tmp = Path(self._tmp.name) + + def tearDown(self): + # 顺序要紧：先还原 env，再清缓存，最后清目录，避免留下一个指向已删除 + # 目录的环境变量给下一个用例。 + if self._env_backup is None: + os.environ.pop(OVERRIDES_ENV, None) + else: + os.environ[OVERRIDES_ENV] = self._env_backup + emitter.clear_price_cache() + self._tmp.cleanup() + + def use_overrides(self, payload, name: str = "overrides.json") -> Path: + """把 payload 写成覆盖文件并指向它（JSON 文本原样写入，便于构造非法输入）。""" + path = self.tmp / name + text = payload if isinstance(payload, str) else json.dumps(payload) + path.write_text(text, encoding="utf-8") + os.environ[OVERRIDES_ENV] = str(path) + emitter.clear_price_cache() + return path + + def pure_builtin(self) -> dict: + """"纯内置表"的唯一权威来源——不把 pricing.json 的内容抄进断言里， + 否则内置表一调整这里就假红；但内置表本身必须是非空的，否则下面的 + "降级后 == 纯内置表" 断言会退化成空表比空表。""" + builtin = emitter._load_builtin_prices() + self.assertTrue(builtin, "内置价格表不应为空，否则降级断言失去意义") + self.assertIn("gpt-4o", builtin) + return builtin + + +class OverridesPathResolutionTests(_OverridesTestCase): + """AC4：路径可覆盖。默认位置刻意放在 skills 树**外**—— + `scripts/build-classic-hosts.py` 会整棵同步 `.codebuddy/skills`，放树内会让 + 用户每次改价都产生一次 `--check` drift，并被复制进 .claude/.cursor 宿主包。""" + + def test_env_var_wins_over_default(self): + path = self.use_overrides({"zz-model": {"output": 1.0}}) + self.assertEqual(emitter.resolve_overrides_path(), path) + + def test_tilde_is_expanded(self): + os.environ[OVERRIDES_ENV] = "~/aobs-overrides-test.json" + emitter.clear_price_cache() + self.assertEqual(emitter.resolve_overrides_path(), Path.home() / "aobs-overrides-test.json") + + def test_unset_falls_back_to_default_path(self): + self.assertEqual(emitter.resolve_overrides_path(), emitter.DEFAULT_OVERRIDES_PATH) + + def test_blank_env_falls_back_to_default(self): + # 空字符串/纯空白都等同于"没设置"，不能变成指向 CWD 的相对路径。 + os.environ[OVERRIDES_ENV] = " " + emitter.clear_price_cache() + self.assertEqual(emitter.resolve_overrides_path(), emitter.DEFAULT_OVERRIDES_PATH) + + def test_default_path_is_outside_the_skills_tree(self): + default = emitter.DEFAULT_OVERRIDES_PATH + self.assertIsNotNone(default) + self.assertEqual( + default, + ROOT.parents[2] / ".codebuddy" / "agent-observability" / "pricing.overrides.json", + ) + self.assertNotIn("skills", default.parts[-3:]) + self.assertFalse(str(default).startswith(str(ROOT))) + + +class MergePricesTests(unittest.TestCase): + """AC2：字段级合并。整 key 覆盖被刻意否决过——漏写字段会把该字段按 0 计， + 静默把成本算没，风险高于收益。""" + + def test_explicit_field_overrides_builtin_while_others_are_kept(self): + merged = emitter.merge_prices( + {"gpt-4o": {"input": 2.5, "output": 10.0, "cache_read": 1.25}}, + {"gpt-4o": {"output": 99.0}}, + ) + self.assertEqual(merged["gpt-4o"]["output"], 99.0) + self.assertEqual(merged["gpt-4o"]["input"], 2.5) + self.assertEqual(merged["gpt-4o"]["cache_read"], 1.25) + + def test_new_model_can_be_added(self): + merged = emitter.merge_prices({"gpt-4o": {"output": 10.0}}, {"zz-new": {"output": 9.0}}) + self.assertEqual(merged["zz-new"], {"output": 9.0}) + + def test_merge_does_not_mutate_its_arguments(self): + base = {"gpt-4o": {"input": 2.5, "output": 10.0}} + overrides = {"gpt-4o": {"output": 99.0}} + merged = emitter.merge_prices(base, overrides) + merged["gpt-4o"]["input"] = 0.0 + # 内置表是模块级共享数据（lru_cache 之外还有调用方持有引用）， + # 被就地改过的话一次合并会污染整条 hook 链路。 + self.assertEqual(base["gpt-4o"], {"input": 2.5, "output": 10.0}) + self.assertEqual(overrides["gpt-4o"], {"output": 99.0}) + + def test_empty_or_none_overrides_return_base_copy(self): + base = {"gpt-4o": {"output": 10.0}} + self.assertEqual(emitter.merge_prices(base, None), base) + self.assertEqual(emitter.merge_prices(base, {}), base) + + def test_none_base_returns_overrides_only(self): + self.assertEqual(emitter.merge_prices(None, {"zz-new": {"output": 1.0}}), {"zz-new": {"output": 1.0}}) + + def test_non_dict_rows_are_ignored(self): + merged = emitter.merge_prices({"gpt-4o": {"output": 10.0}}, {"zz-bad": 9.0, "zz-ok": {"output": 2.0}}) + self.assertNotIn("zz-bad", merged) + self.assertEqual(merged["zz-ok"], {"output": 2.0}) + + +class LoadPricesOverrideTests(_OverridesTestCase): + """AC2 + AC3：加载与降级。""" + + def test_override_applies_field_level_and_adds_model(self): + self.use_overrides({"zz-custom": {"input": 1.0, "output": 9.0}, "gpt-4o": {"output": 99.0}}) + prices = emitter.load_prices() + self.assertEqual(prices["zz-custom"], {"input": 1.0, "output": 9.0}) + self.assertEqual(prices["gpt-4o"]["output"], 99.0) + self.assertEqual(prices["gpt-4o"]["input"], self.pure_builtin()["gpt-4o"]["input"]) + + def test_missing_file_degrades_to_builtin(self): + os.environ[OVERRIDES_ENV] = str(self.tmp / "does-not-exist.json") + emitter.clear_price_cache() + self.assertEqual(emitter.load_prices(), self.pure_builtin()) + + def test_invalid_json_degrades_to_builtin(self): + self.use_overrides('{"gpt-4o": {"output": 99.0}') + self.assertEqual(emitter.load_prices(), self.pure_builtin()) + + def test_top_level_non_dict_degrades_to_builtin(self): + self.use_overrides("[1, 2, 3]") + self.assertEqual(emitter.load_prices(), self.pure_builtin()) + + def test_non_numeric_field_is_skipped_but_siblings_apply(self): + self.use_overrides({"gpt-4o": {"output": "abc", "input": 3.0}}) + prices = emitter.load_prices() + # 写错一个字段不该把同模型其它字段、其它模型的正确覆盖一起丢掉。 + self.assertEqual(prices["gpt-4o"]["input"], 3.0) + self.assertEqual(prices["gpt-4o"]["output"], self.pure_builtin()["gpt-4o"]["output"]) + + def test_model_with_only_bad_fields_is_dropped_and_stays_unpriced(self): + """计划外决策（已在 code review 接受）：全坏字段的模型不落表。 + 若落表成 `{}`，它既算不出成本又不算"未定价"，看板上会变成一个查不到 + 原因的空洞；丢弃后至少能在"模型定价"提醒里暴露出来。""" + self.use_overrides({"zz-all-bad": {"output": "abc"}}) + prices = emitter.load_prices() + self.assertNotIn("zz-all-bad", prices) + self.assertTrue(emitter.is_unpriced("zz-all-bad")) + + def test_bool_is_not_treated_as_a_price(self): + # bool 是 int 的子类，float(True) == 1.0 会静默变成"单价 1 美元"。 + self.use_overrides({"gpt-4o": {"output": True}}) + prices = emitter.load_prices() + self.assertEqual(prices["gpt-4o"]["output"], self.pure_builtin()["gpt-4o"]["output"]) + + def test_every_degradation_path_is_silent(self): + """静默是刻意的：hook 每次都是新进程，一旦因格式问题 print/抛异常， + 整条 hook 链路都会变得不可用。""" + for payload in ('{"broken": ', "[1,2,3]", '{"gpt-4o": {"output": "abc"}}', '{"gpt-4o": 1}'): + with self.subTest(payload=payload): + self.use_overrides(payload) + out, err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + prices = emitter.load_prices() + self.assertEqual(prices, self.pure_builtin()) + self.assertEqual(out.getvalue(), "") + self.assertEqual(err.getvalue(), "") + + def test_load_prices_is_cached_until_cleared(self): + path = self.use_overrides({"zz-cache-model": {"output": 1.0}}) + self.assertEqual(emitter.load_prices()["zz-cache-model"]["output"], 1.0) + path.write_text(json.dumps({"zz-cache-model": {"output": 2.0}}), encoding="utf-8") + # 未清缓存：同进程内仍是旧值（hook 是短命进程，这个行为是可接受的）。 + self.assertEqual(emitter.load_prices()["zz-cache-model"]["output"], 1.0) + emitter.clear_price_cache() + self.assertEqual(emitter.load_prices()["zz-cache-model"]["output"], 2.0) + + def test_clear_price_cache_picks_up_env_change(self): + self.assertTrue(hasattr(emitter.load_prices, "cache_clear")) + self.use_overrides({"zz-a": {"output": 1.0}}) + self.assertIn("zz-a", emitter.load_prices()) + self.use_overrides({"zz-b": {"output": 2.0}}, name="second.json") + emitter.clear_price_cache() + prices = emitter.load_prices() + self.assertIn("zz-b", prices) + self.assertNotIn("zz-a", prices) + + +class IsUnpricedTests(_OverridesTestCase): + """D4：未定价的唯一口径是 `lookup_price(...) is None`。 + `lookup_price` 会做"最长子串"模糊兜底，所以 `gpt-4o-2024-11-20` 能算出成本， + 不该出现在"建议补价"里——口径一旦漂移，用户会去给明明已经命中的模型补价。""" + + def test_unknown_model_is_unpriced(self): + self.assertTrue(emitter.is_unpriced("zz-totally-unknown")) + + def test_fuzzy_substring_match_is_not_unpriced(self): + self.assertFalse(emitter.is_unpriced("gpt-4o-2024-11-20")) + + def test_missing_or_blank_model_is_unpriced(self): + for model in (None, "", " "): + with self.subTest(model=model): + self.assertTrue(emitter.is_unpriced(model)) + + def test_override_makes_a_model_priced(self): + self.assertTrue(emitter.is_unpriced("zz-then-priced")) + self.use_overrides({"zz-then-priced": {"output": 9.0}}) + self.assertFalse(emitter.is_unpriced("zz-then-priced")) + + +class BuildUnpricedModelsTests(_OverridesTestCase): + """AC5：只看板**统计**，绝不回头重算 cost_usd（D3：覆盖只对新事件生效）。""" + + def _usage(self, model, ts=1.0): + return {"event": "usage", "sid": "s1", "ts": ts, "model": model, "tokens": {"input": 1, "output": 1}} + + def test_counts_only_unpriced_usage_events(self): + events = [ + self._usage("zz-unpriced-a", 1.0), + self._usage("zz-unpriced-a", 2.0), + self._usage("zz-unpriced-b", 3.0), + self._usage("gpt-4o-2024-11-20", 4.0), # 模糊命中，不算未定价 + {"event": "usage", "sid": "s1", "ts": 5.0, "tokens": {"input": 1}}, # model 缺失 + self._usage(" ", 6.0), # model 空白 + {"event": "tool", "sid": "s1", "ts": 7.0, "tool": "Bash", "ms": 1}, # 非 usage + ] + rows = bdd.build_unpriced_models(events) + self.assertEqual(rows, [ + {"name": "zz-unpriced-a", "usageEvents": 2}, + {"name": "zz-unpriced-b", "usageEvents": 1}, + ]) + + def test_empty_input_returns_empty_list(self): + self.assertEqual(bdd.build_unpriced_models([]), []) + + def test_sorted_by_event_count_desc_then_name(self): + events = ( + [self._usage("zz-b", float(i)) for i in range(2)] + + [self._usage("zz-a", 9.0)] + + [self._usage("zz-c", float(i)) for i in range(2)] + ) + rows = bdd.build_unpriced_models(events) + # 计数相同的 zz-b / zz-c 必须按名字定序，否则两次生成的快照无法逐字节比对。 + self.assertEqual([(r["name"], r["usageEvents"]) for r in rows], + [("zz-b", 2), ("zz-c", 2), ("zz-a", 1)]) + + def test_pricing_a_model_removes_it_from_the_list(self): + events = [self._usage("zz-unpriced-a", 1.0), self._usage("zz-unpriced-b", 2.0)] + self.assertEqual(len(bdd.build_unpriced_models(events)), 2) + self.use_overrides({"zz-unpriced-a": {"output": 9.0}}) + rows = bdd.build_unpriced_models(events) + self.assertEqual(rows, [{"name": "zz-unpriced-b", "usageEvents": 1}]) + + def test_emitter_import_failure_degrades_to_empty_list(self): + """目标机可能没装 emitter 的间接依赖；此时只该退化"未定价"这一项， + 看板仍要能出片。""" + with mock.patch.object(bdd, "em", None): + self.assertEqual(bdd.build_unpriced_models([self._usage("zz-unpriced-a")]), []) + + +class MainSnapshotTests(_OverridesTestCase): + """main() 级别：键位、空列表、不重算成本、无覆盖时与改动前一致（AC1）。""" + + def _events(self): + return [ + {"event": "user_prompt_submit", "sid": "s1", "ts": 1.0, "turn_id": "t1", "agent": "main"}, + {"event": "usage", "sid": "s1", "ts": 2.0, "model": "gpt-4o-2024-11-20", + "tokens": {"input": 100, "output": 10}, "cost_usd": 0.0035, "agent": "main"}, + {"event": "usage", "sid": "s1", "ts": 3.0, "model": "zz-unpriced-a", + "tokens": {"input": 100, "output": 10}, "agent": "main"}, + {"event": "usage", "sid": "s1", "ts": 4.0, "model": "zz-unpriced-a", + "tokens": {"input": 100, "output": 10}, "agent": "main"}, + ] + + def _run_main(self, root: Path, out_name: str) -> tuple[dict, str]: + metrics = root / "metrics.ndjson" + metrics.write_text("\n".join(json.dumps(e) for e in self._events()) + "\n", encoding="utf-8") + state = root / ".state.json" + state.write_text("{}", encoding="utf-8") + out = root / out_name + old_argv = sys.argv + sys.argv = ["build_dashboard_data.py", "--project-root", str(root), + "--out", str(out), "--metrics-path", str(metrics), "--state-path", str(state)] + buf = io.StringIO() + try: + with contextlib.redirect_stdout(buf): + rc = bdd.main() + finally: + sys.argv = old_argv + self.assertEqual(rc, 0) + self.assertIn("wrote ", buf.getvalue()) + return json.loads(out.read_text("utf-8")), buf.getvalue() + + def test_snapshot_exposes_unpriced_models_right_after_model_costs(self): + with tempfile.TemporaryDirectory() as td: + data, _ = self._run_main(Path(td), "out.json") + keys = list(data.keys()) + self.assertEqual(keys[keys.index("modelCosts") + 1], "unpricedModels") + self.assertEqual(data["unpricedModels"], [{"name": "zz-unpriced-a", "usageEvents": 2}]) + self.assertEqual([m["name"] for m in data["modelCosts"]], ["gpt-4o-2024-11-20"]) + + def test_no_unpriced_models_yields_empty_list_not_null(self): + with tempfile.TemporaryDirectory() as td: + self.use_overrides({"zz-unpriced-a": {"output": 9.0}}) + data, _ = self._run_main(Path(td), "out.json") + self.assertEqual(data["unpricedModels"], []) + + def test_override_never_recomputes_already_written_costs(self): + """D3 的回归护栏：给未定价模型补价后，只有 unpricedModels 会变， + modelCosts / daily / sessions 必须逐字节不变（历史成本不回溯）。""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) + before, _ = self._run_main(root, "before.json") + self.use_overrides({"zz-unpriced-a": {"output": 9.0}}) + after, _ = self._run_main(root, "after.json") + before.pop("generated_at") + after.pop("generated_at") + self.assertEqual(before["unpricedModels"], [{"name": "zz-unpriced-a", "usageEvents": 2}]) + self.assertEqual(after["unpricedModels"], []) + before.pop("unpricedModels") + after.pop("unpricedModels") + self.assertEqual(before, after) + + def test_broken_override_file_keeps_cli_at_exit_zero_and_silent(self): + """AC3 的 CLI 层护栏：覆盖文件坏掉时看板仍然出片（exit 0、stderr 为空）， + 而不是把整条构建链路搞挂。""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) + self.use_overrides('{"gpt-4o": {"output": 99.0}') # 故意截断的非法 JSON + err = io.StringIO() + with contextlib.redirect_stderr(err): + data, _ = self._run_main(root, "out.json") + self.assertEqual(err.getvalue(), "") + self.assertEqual(data["unpricedModels"], [{"name": "zz-unpriced-a", "usageEvents": 2}]) + + def test_no_override_file_matches_nonexistent_override_path(self): + """AC1 的核心回归：不传覆盖文件时价格表就是纯内置表， + 与"指向一个不存在的路径"的输出完全一致（忽略 generated_at）。""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) + unset, _ = self._run_main(root, "unset.json") + os.environ[OVERRIDES_ENV] = str(root / "nope.json") + emitter.clear_price_cache() + missing, _ = self._run_main(root, "missing.json") + unset.pop("generated_at") + missing.pop("generated_at") + self.assertEqual(unset, missing) + self.assertEqual(unset["unpricedModels"], [{"name": "zz-unpriced-a", "usageEvents": 2}]) + + +if __name__ == "__main__": + unittest.main() diff --git a/.codebuddy/skills/agent-observability/tests/test_runtime_flow.py b/.codebuddy/skills/agent-observability/tests/test_runtime_flow.py new file mode 100644 index 0000000..7bd143b --- /dev/null +++ b/.codebuddy/skills/agent-observability/tests/test_runtime_flow.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path +from unittest import mock +import sys + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "scripts" +if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) + +from core import runtime # type: ignore + + +class RuntimeFlowTests(unittest.TestCase): + def test_resolve_active_agent_prefers_inbox_inferred_agent_when_available(self): + data = { + "tool_name": "Task", + "tool_input": {"subagent_name": "developer"}, + } + with tempfile.TemporaryDirectory() as td: + state_path = Path(td) / ".state.json" + with mock.patch.object(runtime.agent_identity, "merge_agent_identity_from_inbox", return_value=("qa", "developer")) as merge_identity: + active, dispatched = runtime.agent_identity.resolve_active_agent_for_event( + state_path=state_path, + sid="s-inbox", + cwd=td, + data=data, + ) + + self.assertEqual(active, "qa") + self.assertEqual(dispatched, "developer") + merge_identity.assert_called_once() + + def test_handle_post_routes_through_expected_collaborators(self): + data = { + "session_id": "s-runtime", + "cwd": "/tmp/demo", + "tool_name": "Read", + "transcript_path": "/tmp/demo.jsonl", + } + with tempfile.TemporaryDirectory() as td: + with mock.patch.object(runtime, "build_runtime_paths") as mock_paths, \ + mock.patch.object(runtime.collector, "record_post", return_value=12), \ + mock.patch.object(runtime.collector, "load_cached_inventory", return_value=({}, {})), \ + mock.patch.object(runtime.scanner, "scan_skills_and_rules", return_value=({}, {})), \ + mock.patch.object(runtime.collector, "cache_inventory"), \ + mock.patch.object(runtime.collector, "record_tool_usage", return_value=([], [])), \ + mock.patch.object(runtime.agent_identity, "resolve_active_agent_for_event", return_value=("main", None)), \ + mock.patch.object(runtime, "current_turn_id", return_value="turn-1"), \ + mock.patch.object(runtime.collector, "extract_tool_call_id", return_value=None), \ + mock.patch.object(runtime, "flush_pending_tool_events"), \ + mock.patch.object(runtime.collector, "find_current_tool_context", return_value=None), \ + mock.patch.object(runtime.collector, "related_transcript_paths", return_value=["/tmp/demo.jsonl"]), \ + mock.patch.object(runtime, "emit_transcript_events", return_value=[]), \ + mock.patch.object(runtime.collector, "find_fallback_usage_event", return_value=None), \ + mock.patch.object(runtime.st, "append_pending_tool_emit"), \ + mock.patch.object(runtime.agentlens, "emit_post_step"): + mock_paths.return_value = runtime.RuntimePaths( + base_dir=Path(td), + state_path=Path(td) / ".state.json", + pending_path=Path(td) / ".pending.json", + log_path=Path(td) / "metrics.ndjson", + ) + runtime.handle_post(data) + + def test_main_dispatches_supported_phase_to_handler(self): + with mock.patch.object(runtime, "handle_post") as handle_post, \ + mock.patch.object(runtime.collector, "read_stdin_json", return_value={"session_id": "s1"}): + rc = runtime.main(["post"]) + self.assertEqual(rc, 0) + handle_post.assert_called_once() diff --git a/.cursor/skills/agent-observability/SKILL.md b/.cursor/skills/agent-observability/SKILL.md new file mode 100644 index 0000000..8bff9f0 --- /dev/null +++ b/.cursor/skills/agent-observability/SKILL.md @@ -0,0 +1,33 @@ +--- +name: agent-observability +description: Add or maintain CodeBuddy hook observability for projects that need local metrics for tool latency, transcript token usage, session cost, and multi-agent trace attribution. Use when wiring hook-based telemetry, debugging missing usage events, and validating AgentLens state. +--- + +## Workflow + +1. Merge `templates/settings-hook.json` into the user or project `settings.json`. +2. Keep hook entries command-only and keep timeouts around 5 to 10 seconds. +3. Run a normal CodeBuddy session and inspect `logs/metrics.ndjson`. +4. Inspect `logs/.state.json` when usage replay, offsets, or AgentLens turn state look wrong. +5. Keep the runtime generic and verify output from `metrics.ndjson` and `.state.json`. + +## Read Next + +- Read `references/quickstart.md` for setup and smoke-test steps. +- Read `references/schema-v2.md` when you need field definitions or sidecar state shape. +## Key Files + +- `scripts/main.py`: hook entrypoint for `session-start`, `user-prompt-submit`, `pre`, `post`, and `stop` +- `scripts/run_hook.sh`: stable shell wrapper for CodeBuddy hook commands +- `scripts/core/collector.py`: transcript parsing, pre/post pairing, and session usage recording +- `scripts/core/agentlens.py`: AgentLens trace emission and agent or step grouping +- `scripts/core/state.py`: persisted hook state, offsets, and AgentLens sidecar state +- `scripts/core/devflow.py`: optional devflow-awareness — detects a `multi-agents-devflow-*` team, reads `workflow-state.json`, and emits `stage_transition` events. No-ops entirely on non-devflow projects. +- `templates/settings-hook.json`: hook wiring template + +## Output Contract + +- Always emit `event`, `sid`, and `ts`. +- Emit `tool` for per-call latency and `usage` for transcript-derived token and cost events. +- Emit `stop` for end-of-session flush. +- Never block the main hook flow; hook exits must stay `0`. diff --git a/.cursor/skills/agent-observability/config/pricing.json b/.cursor/skills/agent-observability/config/pricing.json new file mode 100644 index 0000000..59d8e58 --- /dev/null +++ b/.cursor/skills/agent-observability/config/pricing.json @@ -0,0 +1,39 @@ +{ + "claude-opus-4.8": {"input": 5.0, "output": 25.0, "cache_read": 0.5, "cache_write": 6.25}, + "claude-opus-4.7": {"input": 5.0, "output": 25.0, "cache_read": 0.5, "cache_write": 6.25}, + "claude-opus-4.6": {"input": 5.0, "output": 25.0, "cache_read": 0.5, "cache_write": 6.25}, + "claude-opus-4": {"input": 5.0, "output": 25.0, "cache_read": 0.5, "cache_write": 6.25}, + "claude-sonnet-4.6": {"input": 3.0, "output": 15.0, "cache_read": 0.3, "cache_write": 3.75}, + "claude-sonnet-4.5": {"input": 3.0, "output": 15.0, "cache_read": 0.3, "cache_write": 3.75}, + "claude-sonnet-4": {"input": 3.0, "output": 15.0, "cache_read": 0.3, "cache_write": 3.75}, + "claude-3-7-sonnet": {"input": 3.0, "output": 15.0, "cache_read": 0.3, "cache_write": 3.75}, + "claude-3-5-sonnet": {"input": 3.0, "output": 15.0, "cache_read": 0.3, "cache_write": 3.75}, + "claude-haiku-4.5": {"input": 1.0, "output": 5.0, "cache_read": 0.1, "cache_write": 1.25}, + "claude-3-5-haiku": {"input": 1.0, "output": 5.0, "cache_read": 0.1, "cache_write": 1.25}, + "claude-haiku-4": {"input": 1.0, "output": 5.0, "cache_read": 0.1, "cache_write": 1.25}, + "gpt-5.5-fast": {"input": 12.5, "output": 75.0, "cache_read": 1.25, "cache_write": 12.5}, + "gpt-5.5": {"input": 5.0, "output": 30.0, "cache_read": 0.5, "cache_write": 5.0}, + "gpt-5.4-fast": {"input": 5.0, "output": 30.0, "cache_read": 0.5, "cache_write": 5.0}, + "gpt-5.4": {"input": 2.5, "output": 15.0, "cache_read": 0.25, "cache_write": 2.5}, + "gpt-5.3-codex": {"input": 1.75, "output": 14.0, "cache_read": 0.175, "cache_write": 1.75}, + "gpt-4o": {"input": 2.5, "output": 10.0, "cache_read": 1.25, "cache_write": 2.5}, + "gpt-4o-mini": {"input": 0.15, "output": 0.6, "cache_read": 0.075, "cache_write": 0.15}, + "gemini-3.5-flash": {"input": 1.5, "output": 9.0, "cache_read": 0.15, "cache_write": 1.5}, + "gemini-3.1-pro": {"input": 2.0, "output": 12.0, "cache_read": 0.2, "cache_write": 2.0}, + "gemini-3.1-flash-lite": {"input": 0.25, "output": 1.5, "cache_read": 0.025, "cache_write": 0.25}, + "gemini-3-flash": {"input": 0.5, "output": 3.0, "cache_read": 0.05, "cache_write": 0.5}, + "gemini-2": {"input": 0.5, "output": 3.0, "cache_read": 0.05, "cache_write": 0.5}, + "glm-5.1": {"input": 0.857, "output": 3.429, "cache_read": 0.186, "cache_write": 0.857}, + "glm-5v-turbo": {"input": 0.714, "output": 3.143, "cache_read": 0.171, "cache_write": 0.714}, + "glm-5-turbo": {"input": 0.714, "output": 3.143, "cache_read": 0.171, "cache_write": 0.714}, + "glm-5": {"input": 0.857, "output": 3.429, "cache_read": 0.186, "cache_write": 0.857}, + "kimi-k2.6": {"input": 0.929, "output": 3.857, "cache_read": 0.157, "cache_write": 0.929}, + "kimi-k2.5": {"input": 0.571, "output": 3.0, "cache_read": 0.1, "cache_write": 0.571}, + "kimi": {"input": 0.571, "output": 3.0, "cache_read": 0.1, "cache_write": 0.571}, + "deepseek-v4-pro": {"input": 0.429, "output": 0.857, "cache_read": 0.004, "cache_write": 0.429}, + "deepseek-v4-flash": {"input": 0.143, "output": 0.286, "cache_read": 0.003, "cache_write": 0.143}, + "deepseek-v4": {"input": 0.429, "output": 0.857, "cache_read": 0.004, "cache_write": 0.429}, + "deepseek": {"input": 0.143, "output": 0.286, "cache_read": 0.003, "cache_write": 0.143}, + "minimax-m2.7": {"input": 0.3, "output": 1.2, "cache_read": 0.06, "cache_write": 0.3}, + "minimax": {"input": 0.3, "output": 1.2, "cache_read": 0.06, "cache_write": 0.3} +} diff --git a/.cursor/skills/agent-observability/logs/.gitkeep b/.cursor/skills/agent-observability/logs/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/.cursor/skills/agent-observability/references/quickstart.md b/.cursor/skills/agent-observability/references/quickstart.md new file mode 100644 index 0000000..7a30ef6 --- /dev/null +++ b/.cursor/skills/agent-observability/references/quickstart.md @@ -0,0 +1,195 @@ +## Quickstart + +### 安装 + +将 `templates/settings-hook.json` 合并到 `~/.cursor/settings.json` 或 `/.cursor/settings.json`。 + +必须启用 hooks：`SessionStart`、`UserPromptSubmit`、`PreToolUse`、`PostToolUse`、`Stop`。 + +### 查看输出 + +完成一次正常会话后检查： +- `.cursor/skills/agent-observability/logs/metrics.ndjson` +- `.cursor/skills/agent-observability/logs/.state.json` + +### 指定数据源路径（可选） + +`build_dashboard_data.py` 默认从 `/logs/metrics.ndjson` 与 `/logs/.state.json` 读取，**不传参数时行为完全不变**。 + +如需从其它位置读取，可用 `--metrics-path` / `--state-path` 覆盖： + +```bash +python3 scripts/build_dashboard_data.py \ + --metrics-path ~/Downloads/metrics.ndjson \ + --state-path ~/Downloads/.state.json \ + --out dashboard/dashboard-data.json +``` + +注意： + +- 两个参数都留空（或省略）时回退到 `/logs/` 下的默认文件。 +- 路径支持 `~` 展开（如上例的 `~/Downloads/...`）。 +- 输出 `dashboard-data.json` 的 `source.metrics_ndjson` / `source.state_json` 会**如实反映实际读取到的路径**，覆盖后自然指向你给定的文件，便于核对数据来源。 + +重点字段： +- `event=tool`：工具耗时、`skill`、`rule` +- `event=usage`：`tokens`、`model`、`cost_usd` +- `event=stop`：会话尾部 flush 与总成本 + +### 导出最慢的工具调用（可选） + +`--top-slow N` 会在生成 `dashboard-data.json` 的同时，额外在**终端**打印耗时最长的 N 次 `event=tool` 调用（工具名 + 耗时 ms，按耗时降序）。排查"哪次工具调用拖慢了会话"时不用再去翻原始 ndjson。 + +```bash +python3 scripts/build_dashboard_data.py --top-slow 5 +``` + +输出示例（排在 `wrote ...` 之后）： + +``` +top-slow 5 tool calls (by ms): + 1. Bash 4820ms + 2. Grep 1230ms + 3. Read 310ms +``` + +注意： + +- **默认不开启**：不传该参数时不会打印任何额外内容，原有输出一字不变。 +- **只读**：结果只打印到终端，不会写进 `dashboard-data.json`——输出结构与不开时完全一致，看板不受影响。 +- 只统计 `event=tool` 事件；`ms` 缺失或非数字时按 `0` 兜底，与看板其它统计口径一致。 +- `N` 大于实际 tool 事件数时取全部，不报错；没有任何 tool 事件时不打印该段。 + +### 自定义模型定价覆盖（可选） + +内置价格表 `config/pricing.json` 覆盖不到的模型（自部署模型、内部代号、刚发布还没来得及收录的模型）， +`cost_usd` 会算不出来。给这些模型补单价不用改内置文件——写一份**只含差异**的覆盖文件即可。 + +**放哪**：默认 `/.cursor/agent-observability/pricing.overrides.json` +（`` 是 `.cursor/` 所在的那一层，例如 `/Users/me/my-project`）。 + +刻意放在 `skills/` 树**之外**，原因有三： + +- `scripts/build-classic-hosts.py` 会整棵同步 `.cursor/skills` 到 `.claude/` `.cursor/`， + 放树内会让每次改动都产生 `--check` drift，还会把你的单价复制进生成的宿主包； +- `config/pricing.json` 属于 skill 自带资产，后续更新会把它冲掉，覆盖文件不会； +- 它是纯数据文件，删除或改名即可停用，不需要重启任何东西（hook 每次都是新进程）。 + +**格式**：键是模型名（匹配时忽略大小写与首尾空格），值是要覆盖的字段。 +单位与 `config/pricing.json` 完全一致——**USD / 1M tokens**，可用字段只有四个： +`input`、`output`、`cache_read`、`cache_write`。 + +```json +{ + "my-model": { "output": 9.0 }, + "gpt-4o": { "output": 99.0 } +} +``` + +**合并是字段级的**：上面这份只改 `output`——`my-model` 的 `input`/`cache_read`/`cache_write` +沿用内置值（内置表里没有 `my-model` 时，未覆盖的字段按 0 计），`gpt-4o` 的 `input` 仍是内置的 `2.5`。 +可以新增内置表里不存在的模型，但**不能删除**内置的模型或字段（合并只能加不能减）。 + +**两个环境变量**（都支持 `~` 展开）： + +| 环境变量 | 作用 | 优先级 | +|---|---|---| +| `AOBS_PRICES_PATH` | **替换**整张基础表（不再读内置 `config/pricing.json`） | 低 | +| `AOBS_PRICING_OVERRIDES_PATH` | 在上面那张基础表之上**叠加**一份补丁，指向任意路径 | 高 | + +两者同时设置时，覆盖文件里的字段最终生效。 + +> ⚠️ **只对新事件生效，不回溯**：`cost_usd` 在 hook 期就写进了 `metrics.ndjson`， +> 补价之后只有**之后新产生**的 usage 事件按新价格计算，已经落盘的历史成本不会被重算。 + +**出问题会静默降级**：覆盖文件不存在、JSON 非法、顶层不是对象、字段值不是数字， +四种情况都自动退回纯内置价格表——不报错、不打印、hook 照常退出 0 +（否则一个手误的格式问题会让整条 hook 链路不可用）。所以"改了没生效"通常意味着文件没被读到， +先跑一次诊断： + +```bash +python3 - <<'PY' +import sys +sys.path.insert(0, "scripts") +from core import emitter +print("path :", emitter.resolve_overrides_path()) +print("loaded :", emitter.load_price_overrides()) +print("gpt-4o :", emitter.load_prices().get("gpt-4o")) +PY +``` + +`path` 是实际读取的位置（不是你以为的那个），`loaded` 为空说明文件没读到或全被判为坏字段。 + +**看板上的提示**：「建议关注」卡片第 4 条「模型定价」会列出**完全没命中价格表**的模型 +（模型名 + 缺少成本的 usage 事件数）。模糊匹配（最长子串）命中的模型能算出成本，不会出现在里面。 +处置方式就是把该模型写进覆盖文件；所有模型都有价时这条显示 green。 + +### 常见问题 + +- 没有日志：检查 hook 命令路径。 +- `ms` 为空：通常是 pre/post 未配对。 +- 没有 `usage`：`transcript_path` 缺失、tail 无 usage，或增量已去重。 +- AgentLens 未启用：检查 `.state.json` 的 `_agentlens.enabled` / `last_error`。 + +### 已知问题（未修复） + +- **工具失败事件的 `raw_response` 经常缺失（PostToolUse 与 transcript 落盘之间的时序竞争）**： + 真实 CodeBuddy CLI 场景下实测复现过——一次会故意制造失败的 `Bash` 调用（`exitCode=1`， + transcript 里 `function_call_result.providerData.toolResult.rawResponse` 确实带了 + `is_error:true`/`exitCode:1`/`tool_error_code`），但最终写进 `metrics.ndjson` 的 + `tool` 事件的 `tool_details` 里完全没有 `raw_response` 字段。 + 用实测时间戳定位到根因：PostToolUse hook 记录这次调用的时间是 `ts=...396.524`， + 但 transcript 里 `function_call_result` 真正落盘的时间是 `timestamp=...396.609`—— + **晚了 85ms**。hook 触发时去扫 transcript 文件的那一刻，CLI 还没来得及把执行结果 + 那条记录写盘，`find_current_tool_context()`（`core/collector.py`）只能看到 + `function_call`（请求），看不到 `function_call_result`（结果）。已经单独验证过 + `merge_tool_records()` / `tool_details_from_record()` 的合并逻辑本身没问题—— + 只要数据真的已经在文件里，能正确解析出完整 `raw_response`；问题纯粹是读的时机 + 比 CLI 写盘早了一步。`collect_transcript_entries()` 用的是持久化的增量字节 offset + 游标，错过这次窗口后不会在后续调用里回头补扫，所以这次机会永久丢失，直接后果是 + `build_dashboard_data.py` 的 `tool_call_failed()`（无论怎么改判定逻辑）都拿不到 + 数据，"工具失败率"/`failures` 列表对这类快速失败调用会漏检。 + 复现方式：让 CodeBuddy 执行一个必然快速失败的命令（如 `ls /path/does/not/exist`， + 越快的命令越容易复现，因为 hook 触发与 transcript 落盘之间的竞争窗口更紧张）， + 对比 `metrics.ndjson` 里该 `tool` 事件的 `tool_details.raw_response` 是否存在， + 和对应 transcript `.jsonl` 里 `function_call`/`function_call_result` 两条记录各自 + 的 `timestamp` 先后。 + + **影响范围（已精确定位，不是猜测）**：`raw_response` 在 `build_dashboard_data.py` + 里只有两处消费者——`tool_call_failed()`（喂给 `build_daily_and_sessions()` 的 + `day.failures` 计数和 `session.status`）和 `build_failures()`（"工具失败面板"的 + 数据源）。真实数据统计过：107 次 tool 事件里只有 2 次带 `raw_response`，且只有 + `Bash` 调用会带这个字段（Read/Edit/Write/SendMessage 等其它工具从不带，不受 + 此问题影响）。缺失时 `tool_call_failed()` 默认判"未失败"，所以效果是**恒定 + 漏报，不会误报**——工具失败率/失败面板/会话状态列显示的"正常"可能掩盖了真实 + 发生过的 Bash 失败。turns、duration、dispatch、cost、token、skill/rule 命中、 + devflow 阶段耗时、会话列表本身完全不受影响，是纯观测盲区，不影响 devflow 实际 + 执行行为。 + + **三次修复尝试均已失败，均已回滚（详见下方"已尝试且已放弃的修复方向"）**： + 真正阻塞方向 A 的证据很反常——三次独立测试里，预算从 160ms 加到 500ms 再加到 + 2000ms，实测缺口每次都精确地"比预算多几十到一百多毫秒"（206/564/2151ms）， + 不像是在等一个独立发生的固定延迟，更像是**hook 自己的同步等待在阻塞 CLI 落盘** + ——等得越久，结果来得越晚。这个因果关系还没验证清楚，在验证清楚之前，继续在 + hook 里加同步等待大概率是死路，不建议再尝试。 + +### 已尝试且已放弃的修复方向（供以后参考，避免重复踩坑） + +1. **方向 A：PostToolUse 里同步有界重试**（`claim=False` 轮询直到等到 `raw_response` + 或超时，只对 `tool_name=="Bash"` 生效）。三次真实端到端验证，预算 160ms/500ms/ + 2000ms 全部失败，且"缺口≈预算+常数"的规律强烈暗示等待本身可能在拖慢 CLI + 落盘（见上文）。不建议在搞清楚这层因果关系之前继续加大预算。 +2. **方向 B：推迟到 `pending_tool_emits` 重试队列，下次 hook 触发或 session Stop + 时再补**（`claim=False` 探测 + 延后 `claim=True`，避免过早消费掉 call_id）。 + 逻辑和单元测试都通过，但端到端验证暴露了一个更深的、独立于这次修复的既有 + 架构问题：`find_current_tool_context()`（工具上下文查询）和 + `emit_transcript_events()`（usage token 扫描）共用同一个持久化字节偏移游标 + （`core/state.py` 里只按 `(sid, transcript_path)` 区分，不分用途）。 + `emit_transcript_events()` 每次 `handle_post()` 都无条件推进这个游标，一旦 + 推过某段内容，后续任何工具上下文重试在这段范围内都会**彻底找不到任何数据** + （不只是缺 `raw_response`，连 `call_id`/`arguments` 都没了）——比不修复更糟。 + 真实验证过两次：两条端到端测试调用最终落盘时 `tool_details`完全是空的。 + 要让方向 B 真正可行，必须先给工具上下文查询一个独立于 usage 扫描的游标， + 这是范围更大、需要认真设计的改动，还没有细化方案。 + +以上两个方向的实现和回滚记录详见会话 memory（`project-agent-observability-raw-response-race`）。 diff --git a/.cursor/skills/agent-observability/references/schema-v2.md b/.cursor/skills/agent-observability/references/schema-v2.md new file mode 100644 index 0000000..a04d4d0 --- /dev/null +++ b/.cursor/skills/agent-observability/references/schema-v2.md @@ -0,0 +1,282 @@ +## Schema v2.3 + +`agent-observability` 输出 `ndjson`，每行一个事件对象。 + +> 以下字段表基于当前 `logs/metrics.ndjson` 的实际输出整理；其中“可选字段”只会在特定场景出现。 + +### 通用字段 + +- `event`: `start | user_prompt_submit | tool | usage | stop | error | stage_transition` +- `sid`: session id +- `ts`: 秒级时间戳（float） +- `turn_id`: 可选；由 `UserPromptSubmit` 生成，贯穿本轮事件 + +### Devflow 感知字段（可选，见 `core/devflow.py`） + +只有当前项目正在跑 `.cursor/runtime` 描述的 multi-agents-devflow 工作流（存在 +`.cursor/teams/multi-agents-devflow-{task_slug}/` team 目录）时才会出现，非 +devflow 项目完全不受影响： + +- `tool` / `usage` / `stop` 事件上会附带 `task_slug`（devflow 需求标识，用于把 + main + 最多 7 个常驻角色跨事件串成"同一次运行"；⚠️ `/resume-devflow` 中断恢复 + 可能发生在新的 CodeBuddy 顶层 session 里，因此**聚合一次 devflow 运行要按 + `task_slug` 分组，不能按 `sid` 分组**）与 `stage`（`workflow-state.json` 的 + `current_stage`，如 `TASK-03`/`CODE-REVIEW`）。 +- `tool` 事件在能从 subagent transcript 文件名识别出并行 sub-developer 轨道时 + （`sub-developer-PT-01` 这类命名）会额外带 `pt_id`（如 `"PT-01"`）。按事件自己的 + transcript_path 推断，不依赖任何 session 级共享状态——并行轨道是真并发，不能 + 用一个可变指针记"当前是哪条轨道"。 + +### 事件字段 + +| event | 实际顶层字段 | 说明 | +|---|---|---| +| `start` | `event`, `sid`, `agent`, `ts` | 会话开始 | +| `user_prompt_submit` | `event`, `sid`, `agent`, `turn_id`, `prompt_len`, `ts` | 新 turn 边界；AgentLens 侧按 `1 Trace = 1 Turn` 生成 trace | +| `tool` | `event`, `sid`, `agent`, `tool`, `ms`, `transcript_path`, `turn_id`, `ts`, `skill`, `rule`, `cwd`, `call_id`, `message_id`, `tool_details`, `task_slug`?, `stage`?, `pt_id`? | 工具调用与耗时 | +| `usage` | `event`, `sid`, `agent`, `tool`, `tokens`, `model`, `transcript_path`, `source_offset`, `turn_id`, `message_id`, `ts`, `task_slug`?, `stage`? | transcript 增量 token / 模型 / 消息归属 | +| `stop` | `event`, `sid`, `agent`, `tokens`, `model`, `transcript_path`, `source_offset`, `turn_id`, `message_id`, `ts`, `task_slug`?, `stage`? | 会话尾部 flush | +| `error` | `event`, `sid`, `phase`, `error`, `x_traceback`, `ts` | hook 自身异常记录 | +| `stage_transition` | `event`, `sid`, `task_slug`, `stage`, `status`?, `executor`?, `retry_count`?, `review_result`?, `ts` | devflow `workflow-state.json` 某个 stage 的 status/retry_count/review_result 发生变化时触发（仅 devflow 项目） | + +### 可选字段说明 + +- `tool.ms`: Pre/Post 未成功配对时可能为 `null` +- `tool.call_id`: 仅当工具调用存在 call id 时出现 +- `tool.message_id`: 仅当成功和 transcript 中的 message 关联上时出现 +- `tool.tool_details`: 仅当 transcript 中能还原出更细工具上下文时出现 +- `usage.message_id`: 仅当 usage 对应的 transcript message 可识别时出现 +- `usage.model`: transcript 中能识别模型名时出现 +- `usage.cost_usd`: 只有模型价格命中价格表时才会出现。价格表 = 内置 `config/pricing.json` 与用户覆盖文件（默认 `/.cursor/agent-observability/pricing.overrides.json`，可用 `AOBS_PRICING_OVERRIDES_PATH` 指定）的字段级合并结果；覆盖只对该文件写入**之后**产生的新事件生效，不回溯重算历史成本（见 `quickstart.md` 的「自定义模型定价覆盖」） +- `stop.cost_usd`: 只有当前 stop 事件对应 usage 能估算成本时才会出现 +- `stop.cost_session_usd`: 只有 stop 汇总阶段能反算出整个 session 成本时才会出现 + +### 完整示例 + +换成更容易读的多行 JSON。下面仍然是当前 `metrics.ndjson` 里的实际数据，只是长文本字段做了截断。 + +`start` + +```json +{ + "event": "start", + "sid": "f6435d63-b4d5-44f2-b4d6-ae73d6c140fd", + "agent": "main", + "ts": 1784015331.87806 +} +``` + +`user_prompt_submit` + +```json +{ + "event": "user_prompt_submit", + "sid": "f6435d63-b4d5-44f2-b4d6-ae73d6c140fd", + "agent": "main", + "turn_id": "turn-1784015342075", + "prompt_len": 136, + "ts": 1784015342.076438 +} +``` + +`usage` + +```json +{ + "event": "usage", + "sid": "f6435d63-b4d5-44f2-b4d6-ae73d6c140fd", + "agent": "main", + "tool": "Read", + "tokens": { + "input": 31110, + "output": 594, + "cache_read": 3072, + "total": 31704 + }, + "model": "hy3-ioa", + "transcript_path": "/Users/rachel/.cursor/projects/Users-rachel-skillhub-tokentrack-mr/f6435d63-b4d5-44f2-b4d6-ae73d6c140fd.jsonl", + "source_offset": 17423, + "turn_id": "turn-1784015342075", + "message_id": "d187771c6eef4520a530564dd2a73e38", + "ts": 1784015355.413464 +} +``` + +`tool` 调用参数型 + +```json +{ + "event": "tool", + "sid": "f6435d63-b4d5-44f2-b4d6-ae73d6c140fd", + "agent": "main", + "tool": "Read", + "ms": null, + "transcript_path": "/Users/rachel/.cursor/projects/Users-rachel-skillhub-tokentrack-mr/f6435d63-b4d5-44f2-b4d6-ae73d6c140fd.jsonl", + "turn_id": "turn-1784015342075", + "ts": 1784015355.412617, + "skill": [], + "rule": [ + "global" + ], + "cwd": "/Users/rachel/skillhub-tokentrack-mr", + "call_id": "chatcmpl-tool-9e50f7ec1d7e4a27", + "message_id": "d187771c6eef4520a530564dd2a73e38", + "tool_details": { + "call_id": "chatcmpl-tool-9e50f7ec1d7e4a27", + "arguments": "{\"file_path\": \"/Users/rachel/skillhub-tokentrack-mr/assets/devflow.defaults.yaml\"}", + "arguments_display_text": "assets/devflow.defaults.yaml" + } +} +``` + +`tool` 返回结果型 + +```json +{ + "event": "tool", + "sid": "f6435d63-b4d5-44f2-b4d6-ae73d6c140fd", + "agent": "main", + "tool": "Bash", + "ms": 759, + "transcript_path": "/Users/rachel/.cursor/projects/Users-rachel-skillhub-tokentrack-mr/f6435d63-b4d5-44f2-b4d6-ae73d6c140fd.jsonl", + "turn_id": "turn-1784015342075", + "ts": 1784015356.071573, + "skill": [], + "rule": [ + "global" + ], + "cwd": "/Users/rachel/skillhub-tokentrack-mr", + "call_id": "chatcmpl-tool-898d45e0999b3e3c", + "message_id": "93b617ceebaa49458169b89b4ecae17b", + "tool_details": { + "call_id": "chatcmpl-tool-898d45e0999b3e3c", + "result_content": "Command: cd /Users/rachel/skillhub-tokentrack-mr && ls -la .cursor/teams/ ...", + "raw_response": { + "exitCode": 1, + "signal": null, + "interrupted": false, + "sandboxDenied": false, + "stderrBytesTruncated": 0, + "stdoutBytesTruncated": 0, + "tool_error_code": "8002", + "is_error": true, + "error": "Command: cd /Users/rachel/skillhub-tokentrack-mr && ls -la .cursor/teams/ ..." + }, + "output_text": "Command: cd /Users/rachel/skillhub-tokentrack-mr && ls -la .cursor/teams/ ...", + "original_message_id": "d187771c6eef4520a530564dd2a73e38", + "next_message_id": "93b617ceebaa49458169b89b4ecae17b", + "message_id_reassigned": true + } +} +``` + +`stop` + +```json +{ + "event": "stop", + "sid": "f6435d63-b4d5-44f2-b4d6-ae73d6c140fd", + "agent": "test-engineer", + "tokens": { + "input": 128290, + "output": 665, + "cache_read": 128192, + "total": 128955 + }, + "model": "hy3-ioa", + "transcript_path": "/Users/rachel/.cursor/projects/Users-rachel-skillhub-tokentrack-mr/f6435d63-b4d5-44f2-b4d6-ae73d6c140fd.jsonl", + "source_offset": 1157291, + "turn_id": "turn-1784015342075", + "message_id": "51e4fae9553d4d0995e7a599f3b2ef2d", + "ts": 1784018405.467215 +} +``` + +`error` + +```json +{ + "event": "error", + "sid": "6e8cd633-1a2f-4488-b1df-1eef168009c3", + "phase": "post", + "error": "AttributeError: module 'core.transcript_runtime' has no attribute 'transcript_path'", + "x_traceback": "Traceback ...", + "ts": 1784183163.4314518 +} +``` + +`stage_transition`（仅 devflow 项目；见 `core/devflow.py`） + +```json +{ + "event": "stage_transition", + "sid": "f6435d63-b4d5-44f2-b4d6-ae73d6c140fd", + "task_slug": "fix-token-bypass_20260911_0900", + "stage": "CODE-REVIEW", + "status": "failed", + "executor": "code-reviewer", + "retry_count": 1, + "review_result": "failed", + "ts": 1784015412.223 +} +``` + +### tokens + +```json +{"input": 1200, "output": 180, "cache_read": 9000, "cache_creation": 0, "total": 1380} +``` + +- `total`: 统一按 `input + output` 计算 +- `cache_read` / `cache_creation`: 保留给成本估算和缓存命中分析使用 + +### tool_details + +`tool` 事件里的 `tool_details` 是一个可选嵌套对象，当前实现里可能包含： + +- `call_id` +- `arguments` +- `arguments_display_text` +- `result_content` +- `raw_response` +- `output_text` +- `original_message_id` +- `next_message_id` +- `message_id_reassigned` + +常见示例： + +```json +{ + "call_id": "chatcmpl-tool-9e50f7ec1d7e4a27", + "arguments": "{\"file_path\": \"/path/to/file\"}", + "arguments_display_text": "path/to/file" +} +``` + +### AgentLens sidecar + +`.state.json[sid]._agentlens` 只作为内部状态使用，核心字段： + +- `enabled`: AgentLens 上报是否可用 +- `last_error`: 最近一次降级原因 +- `current_turn.carrier.traceparent`: 当前 turn 的 trace context +- `current_turn.subagent_spans`: 同一 turn 下的子 agent span carrier +- `current_turn.agent_spans`: 同一 turn 下按 agent 聚合的 span carrier 与累计统计 +- `current_turn.step_spans`: `(agent, message_id)` 级 step span registry +- `turn_history`: 已结束 turn 的摘要 + +### Hook 接入 + +参考 `templates/settings-hook.json`，必须启用： + +- `SessionStart` +- `UserPromptSubmit` +- `PreToolUse` +- `PostToolUse` +- `Stop` + +### 兼容 + +- 旧日志若在 `tool` 事件里直接携带 `tokens` / `cost_usd`，按 legacy usage 处理。 +- 不理解 `turn_id` 的下游消费者可安全忽略。 diff --git a/.cursor/skills/agent-observability/scripts/__init__.py b/.cursor/skills/agent-observability/scripts/__init__.py new file mode 100644 index 0000000..52073be --- /dev/null +++ b/.cursor/skills/agent-observability/scripts/__init__.py @@ -0,0 +1 @@ +"""agent-observability 的脚本包。""" diff --git a/.cursor/skills/agent-observability/scripts/build_dashboard_data.py b/.cursor/skills/agent-observability/scripts/build_dashboard_data.py new file mode 100644 index 0000000..e33f816 --- /dev/null +++ b/.cursor/skills/agent-observability/scripts/build_dashboard_data.py @@ -0,0 +1,624 @@ +#!/usr/bin/env python3 +"""把 `logs/metrics.ndjson` + `artifacts/*/workflow-state.json` + `.state.json` + +`hooks/logs/auto-dispatch.log` 聚合成看板需要的一份 `dashboard-data.json` 快照。 + +这是本地看板的数据源，不是 hook 链路的一部分——hook 只管往 metrics.ndjson 追加事件， +这个脚本单独、按需运行（比如每次想看一眼看板之前手动跑一次，或者配合文件监听器）。 +复用 core.devflow 的 stage_snapshot 做 workflow-state.json 的 schema 归一化， +不在这里重新实现一遍 Classic/Portable 的字段差异。 + +用法： + python3 build_dashboard_data.py --project-root <项目根目录> --out dashboard/dashboard-data.json + python3 build_dashboard_data.py --metrics-path ~/other/metrics.ndjson --state-path ~/other/.state.json + python3 build_dashboard_data.py --top-slow 5 + +不传 --metrics-path / --state-path 时，仍从 /logs/ 下的默认文件读取； +两者均支持 ~ 展开，输出 source 会如实反映实际读取路径。 + +--top-slow N 默认不开启，开启后只在终端额外打印耗时最长的 N 次 tool 事件， +不写入 dashboard-data.json（输出结构与不开时完全一致）。 + +没有任何真实数据时（hook 刚接上、还没跑过 session），会输出一份全空但结构合法的快照， +不会报错、也不会伪造数据。 +""" +from __future__ import annotations + +import argparse +import json +import sys +from collections import defaultdict +from pathlib import Path +from typing import Any + +SCRIPTS_DIR = Path(__file__).resolve().parent +if str(SCRIPTS_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPTS_DIR)) + +from core import devflow as dv # type: ignore + +try: # emitter 只在"未定价模型"统计里用到，导入失败只让这一项退化为 [] + from core import emitter as em # type: ignore +except Exception: # pragma: no cover - 目标机缺依赖时的兜底 + em = None # type: ignore + +# devflow 两套 schema 的 stage 名 -> 展示用 (label, name)。取不到的 stage 用 key 本身兜底。 +STAGE_META: dict[str, tuple[str, str]] = { + "PHASE-0": ("P0", "初始化 + 判定大小"), + "SOLO": ("SOLO", "单 agent 全流程"), + "TASK-01": ("T01", "需求分析 + 澄清"), + "REQUIREMENT": ("REQ", "需求分析 + 澄清"), + "TASK-02": ("T02", "技术方案"), + "DESIGN": ("DES", "技术方案"), + "TASK-03": ("T03", "代码实现"), + "IMPLEMENT": ("IMPL", "代码实现"), + "CODE-REVIEW": ("CR", "代码审查"), + "REVIEW": ("REV", "代码审查"), + "TASK-04": ("T04", "E2E 测试"), + "TEST": ("TEST", "E2E 测试"), + "TASK-05": ("T05", "知识沉淀"), + "KNOWLEDGE": ("KNOW", "知识沉淀"), + "SUMMARY": ("汇总", "最终汇总"), +} +# 两套 schema 各自的阶段顺序，用来按正确顺序渲染 stepper（不能直接遍历 dict，顺序不保证）。 +CLASSIC_ORDER = ["PHASE-0", "TASK-01", "TASK-02", "TASK-03", "CODE-REVIEW", "TASK-04", "TASK-05"] +# SOLO 完成后，small 任务会由 solo-developer 合并执行 TASK-05 知识沉淀（真实运行验证过， +# 不是理论上可选的分支）；stepper 顺序必须把它列进去，否则会把已完成的阶段悄悄漏掉。 +CLASSIC_SOLO_ORDER = ["PHASE-0", "SOLO", "TASK-05"] +PORTABLE_ORDER = ["PHASE-0", "REQUIREMENT", "DESIGN", "IMPLEMENT", "REVIEW", "TEST", "KNOWLEDGE", "SUMMARY"] +PORTABLE_SOLO_ORDER = ["PHASE-0", "SOLO", "SUMMARY"] + + +def read_ndjson(path: Path) -> list[dict[str, Any]]: + records: list[dict[str, Any]] = [] + if not path.is_file(): + return records + with path.open("r", encoding="utf-8") as fp: + for line in fp: + line = line.strip() + if not line.startswith("{"): + continue + try: + obj = json.loads(line) + except Exception: + continue + if isinstance(obj, dict): + records.append(obj) + return records + + +def safe_load_json(path: Path) -> Any: + try: + return json.loads(path.read_text("utf-8")) + except Exception: + return None + + +def day_of(ts: float) -> str: + from datetime import datetime, timezone + return datetime.fromtimestamp(ts, tz=timezone.utc).strftime("%Y-%m-%d") + + +def tool_call_failed(raw_response: Any) -> bool: + """判断一次工具调用是否失败。 + + 真实 CodeBuddy CLI 的 transcript（`function_call_result.providerData.toolResult. + rawResponse`）里从来没有 `is_error` 这个布尔字段——实际信号是 `exitCode`（非 0 + 即失败）和/或 `tool_error_code`（非 "0"/空即失败）。之前只认 `is_error`，导致 + 失败统计在这个宿主上永远是 0，不管命令是否真的失败（复合命令比如 + `cmd; echo ...` 会把失败进一步掩盖成 exitCode=0，那种情况下这里也如实判定为 + 未失败——判断整条工具调用本身有没有失败，不追究命令内部的子步骤）。 + 仍然保留 `is_error` 判断，兼容其它可能真的写这个字段的宿主。 + """ + if not isinstance(raw_response, dict): + return False + if raw_response.get("is_error"): + return True + exit_code = raw_response.get("exitCode") + if isinstance(exit_code, (int, float)) and exit_code != 0: + return True + tool_error_code = raw_response.get("tool_error_code") + if isinstance(tool_error_code, str) and tool_error_code.strip() not in ("", "0"): + return True + return False + + +def build_daily_and_sessions(events: list[dict[str, Any]]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + by_day: dict[str, dict[str, Any]] = {} + by_sid: dict[str, dict[str, Any]] = {} + by_sid_turn: dict[tuple[str, str], dict[str, Any]] = {} + # AgentLens（写 state.current_turn 的那套 tracing）默认关闭时，tool/usage 事件 + # 自身的 turn_id 永远是 None——但 user_prompt_submit 事件不依赖 AgentLens， + # 始终携带真实 turn_id（见 emitter.build_prompt_submit_event）。按 ts 排序后， + # 把它当作 turn 边界，让后续没有自带 turn_id 的 tool/usage 事件归入"当前 sid + # 最近一次打开的 turn"，而不是全部塌缩进同一个 "?" 占位桶。 + current_turn_by_sid: dict[str, str] = {} + + def day_bucket(iso: str) -> dict[str, Any]: + return by_day.setdefault(iso, { + "iso": iso, "sessionCount": 0, "input": 0, "output": 0, "cache": 0, + "cost": 0.0, "toolCalls": 0, "failures": 0, "skillHits": defaultdict(int), + "_sids": set(), + }) + + def sess_bucket(sid: str) -> dict[str, Any]: + return by_sid.setdefault(sid, { + "id": sid, "date": None, "agents": set(), "turns": set(), "toolCalls": 0, + "tokens": 0, "cost": 0.0, "first_ts": None, "last_ts": None, "models": defaultdict(int), + "status": "ok", "timeline": {}, + }) + + for rec in sorted(events, key=lambda r: r.get("ts") if isinstance(r.get("ts"), (int, float)) else 0): + event = rec.get("event") + sid = str(rec.get("sid") or "") + ts = rec.get("ts") + if not sid or not isinstance(ts, (int, float)): + continue + iso = day_of(ts) + sess = sess_bucket(sid) + sess["_seen_day"] = iso + # 不管事件类型，只要这天有这个 sid 的任何事件就算一次"当日活跃会话"。 + # 之前只在 tool/usage 分支里 add，纯对话（只有 user_prompt_submit，没有 + # 触发任何工具调用也没有 usage）的会话永远不会被计入任何一天的 + # sessionCount——总览页的"会话数"会比"会话浏览"tab 里实际展开的会话数少 + # （真实数据复现过：5 个 session 只统计出 4 个 sessionCount）。 + day_bucket(iso)["_sids"].add(sid) + if sess["first_ts"] is None or ts < sess["first_ts"]: + sess["first_ts"] = ts + sess["date"] = iso + if sess["last_ts"] is None or ts > sess["last_ts"]: + sess["last_ts"] = ts + agent = rec.get("agent") + if agent: + sess["agents"].add(str(agent)) + raw_turn_id = rec.get("turn_id") + if event == "user_prompt_submit" and raw_turn_id: + current_turn_by_sid[sid] = str(raw_turn_id) + # 提前建桶——纯对话轮次（没有触发任何工具调用，只有 usage 都没有） + # 之前只在 tool/usage 分支里 setdefault，这种轮次永远不会出现在 + # timeline 里，导致 turns 计数和时间线展开的分组数对不上。 + by_sid_turn.setdefault( + (sid, str(raw_turn_id)), + {"turn": raw_turn_id, "agent": agent or "main", "events": []}, + ) + turn_id = raw_turn_id or current_turn_by_sid.get(sid) + if turn_id: + sess["turns"].add(str(turn_id)) + + if event == "tool": + day = day_bucket(iso) + day["toolCalls"] += 1 + sess["toolCalls"] += 1 + for s in rec.get("skill") or []: + day["skillHits"][str(s)] += 1 + tool_details = rec.get("tool_details") or {} + raw_response = tool_details.get("raw_response") if isinstance(tool_details, dict) else None + is_err = tool_call_failed(raw_response) + if is_err: + day["failures"] += 1 + sess["status"] = "error" + key = (sid, str(turn_id or "?")) + turn = by_sid_turn.setdefault(key, {"turn": turn_id or "?", "agent": agent or "main", "events": []}) + turn["events"].append({ + "kind": "tool", "tool": rec.get("tool"), + "ms": rec.get("ms") if isinstance(rec.get("ms"), (int, float)) else 0, + "err": is_err, + "agent": str(agent) if agent else "main", + }) + elif event == "usage": + day = day_bucket(iso) + tokens = rec.get("tokens") or {} + day["input"] += int(tokens.get("input") or 0) + day["output"] += int(tokens.get("output") or 0) + day["cache"] += int(tokens.get("cache_read") or 0) + cost = rec.get("cost_usd") + if isinstance(cost, (int, float)): + day["cost"] += float(cost) + sess["cost"] += float(cost) + total_tok = int(tokens.get("input") or 0) + int(tokens.get("output") or 0) + sess["tokens"] += total_tok + model = rec.get("model") + if model: + sess["models"][str(model)] += 1 + key = (sid, str(turn_id or "?")) + turn = by_sid_turn.setdefault(key, {"turn": turn_id or "?", "agent": agent or "main", "events": []}) + turn["events"].append({ + "kind": "usage", "tokens": total_tok, + "agent": str(agent) if agent else "main", + }) + elif event == "error": + sess["status"] = "error" + + for (sid, _turn_key), turn in by_sid_turn.items(): + sess = by_sid.setdefault(sid, sess_bucket(sid)) + sess["timeline"].setdefault(_turn_key, turn) + + daily = [] + for iso in sorted(by_day.keys()): + d = by_day[iso] + daily.append({ + "iso": iso, "sessionCount": len(d["_sids"]), "input": d["input"], "output": d["output"], + "cache": d["cache"], "cost": round(d["cost"], 4), "toolCalls": d["toolCalls"], + "failures": d["failures"], "skillHits": dict(d["skillHits"]), + }) + + sessions = [] + for sid, s in by_sid.items(): + if s["first_ts"] is None: + continue + duration = int((s["last_ts"] or s["first_ts"]) - s["first_ts"]) + model = max(s["models"].items(), key=lambda kv: kv[1])[0] if s["models"] else None + timeline_sorted = sorted(s["timeline"].values(), key=lambda t: str(t.get("turn") or "")) + for idx, t in enumerate(timeline_sorted, start=1): + t["turn"] = idx + sessions.append({ + "id": sid, "date": s["date"], "agent": ", ".join(sorted(s["agents"])) or "main", + "turns": len(s["turns"]) or len(timeline_sorted), "toolCalls": s["toolCalls"], + "tokens": s["tokens"], "cost": round(s["cost"], 4), "duration": duration, + "model": model, "status": s["status"], "timeline": timeline_sorted, + }) + sessions.sort(key=lambda s: s["date"] or "", reverse=True) + return daily, sessions + + +def build_model_costs(events: list[dict[str, Any]]) -> list[dict[str, Any]]: + totals: dict[str, float] = defaultdict(float) + for rec in events: + if rec.get("event") != "usage": + continue + model = rec.get("model") + cost = rec.get("cost_usd") + if model and isinstance(cost, (int, float)): + totals[str(model)] += float(cost) + rows = [{"name": name, "cost": round(cost, 4)} for name, cost in totals.items()] + rows.sort(key=lambda r: r["cost"], reverse=True) + return rows + + +def build_unpriced_models(events: list[dict[str, Any]]) -> list[dict[str, Any]]: + """聚合"完全没命中价格表"的模型（D4：`emitter.is_unpriced(model)` 为 True）。 + + 只**统计**不重算：`cost_usd` 在 hook 期就写进事件了，这里不会（也不该）回头 + 按新价格补算历史事件（D3：覆盖只对新事件生效）。判定入口复用 emitter， + 不在本文件里重写一套模型名匹配逻辑，避免与 hook 期的口径分叉。 + + 模型名缺失/为空的 usage 事件无法归因到某个模型，直接跳过不计数。 + """ + if em is None: + return [] + counts: dict[str, int] = defaultdict(int) + for rec in events: + if rec.get("event") != "usage": + continue + model = rec.get("model") + if not model or not str(model).strip(): + continue + name = str(model) + try: + if em.is_unpriced(name): + counts[name] += 1 + except Exception: + continue + rows = [{"name": name, "usageEvents": count} for name, count in counts.items()] + # 按 (-事件数, 名字) 排序：避免看板顺序抖动，两次生成的快照可直接逐字节比对。 + rows.sort(key=lambda r: (-r["usageEvents"], r["name"])) + return rows + + +def build_failures(events: list[dict[str, Any]]) -> list[dict[str, Any]]: + agg: dict[tuple[str, str], dict[str, Any]] = {} + for rec in events: + if rec.get("event") != "tool": + continue + tool_details = rec.get("tool_details") or {} + raw_response = tool_details.get("raw_response") if isinstance(tool_details, dict) else None + if not tool_call_failed(raw_response): + continue + tool = str(rec.get("tool") or "unknown") + code = str(raw_response.get("tool_error_code") or raw_response.get("exitCode") or "error") + key = (tool, code) + row = agg.setdefault(key, {"tool": tool, "code": code, "count": 0, "last": None, "_last_ts": 0.0}) + row["count"] += 1 + ts = rec.get("ts") + if isinstance(ts, (int, float)) and ts > row["_last_ts"]: + row["_last_ts"] = ts + row["last"] = day_of(ts) + rows = list(agg.values()) + for row in rows: + row.pop("_last_ts", None) + rows.sort(key=lambda r: r["count"], reverse=True) + return rows + + +def build_top_slow(events: list[dict[str, Any]], limit: int | None = None) -> list[dict[str, Any]]: + """取耗时最长的 N 次 tool 事件，按 ms 降序。 + + 只做"读"：不改动 events，也不参与 dashboard-data.json 的任何字段——调用方拿 + 结果去打印即可。`ms` 的兜底口径与 build_daily_and_sessions 保持一致（缺失或 + 非数字按 0 计），否则同一份日志会得出两个互相矛盾的工具耗时视图。 + """ + if not isinstance(limit, int) or limit <= 0: + return [] + rows: list[dict[str, Any]] = [] + for rec in events: + if rec.get("event") != "tool": + continue + rows.append({ + "tool": str(rec.get("tool") or "unknown"), + "ms": rec.get("ms") if isinstance(rec.get("ms"), (int, float)) else 0, + }) + rows.sort(key=lambda r: r["ms"], reverse=True) + return rows[:limit] + + +def format_top_slow(rows: list[dict[str, Any]]) -> str: + """把 top-slow 结果渲染成终端文本；没有可展示的行时返回空串（由调用方决定 + 是否打印，避免没有任何 tool 事件时空打一个表头）。""" + if not rows: + return "" + width = max(len(str(r["tool"])) for r in rows) + lines = [f"top-slow {len(rows)} tool calls (by ms):"] + for idx, row in enumerate(rows, start=1): + lines.append(f" {idx}. {str(row['tool']):<{width}} {row['ms']}ms") + return "\n".join(lines) + + +def build_skills(state: dict[str, Any]) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + totals: dict[str, int] = defaultdict(int) + never_used: dict[str, dict[str, Any]] = {} + for sid, sess in state.items(): + if sid.startswith("_") or not isinstance(sess, dict): + continue + for name, rec in (sess.get("skills") or {}).items(): + if not isinstance(rec, dict): + continue + totals[name] += int(rec.get("count") or 0) + for name, rec in (sess.get("rules") or {}).items(): + if not isinstance(rec, dict) or int(rec.get("count") or 0) > 0: + continue + never_used[name] = {"name": name, "reason": "no_paths", "detail": "从未被路径推断或 active-rule 命中"} + for name, count in totals.items(): + if count == 0: + never_used.setdefault(name, {"name": name, "reason": "no_paths", "detail": "静态扫描到，但从未被任何工具调用命中"}) + skill_rows = [{"name": name, "count": count} for name, count in totals.items() if count > 0] + skill_rows.sort(key=lambda r: r["count"], reverse=True) + return skill_rows, sorted(never_used.values(), key=lambda r: r["name"]) + + + +# "team-lead" 是 CodeBuddy 原生 team 基础设施里 lead/orchestrator session 自己的 +# mailbox 名字，语义上就是 main（core/agent_identity.py::normalize_role_name 把它 +# 和 "main" 归为同一类）；不是一个真实存在的子 agent，不应出现在"派发目标"里。 +_DISPATCH_EXCLUDE_AGENTS = {"main", "team-lead"} + + +def build_dispatch(state: dict[str, Any]) -> list[dict[str, Any]]: + totals: dict[str, int] = defaultdict(int) + for sid, sess in state.items(): + if sid.startswith("_") or not isinstance(sess, dict): + continue + for entry in sess.get("agent_history") or []: + if not isinstance(entry, dict): + continue + evidence = str(entry.get("evidence") or "") + agent = entry.get("agent") + if not agent or agent in _DISPATCH_EXCLUDE_AGENTS: + continue + # 只统计真正代表"新派发"的证据：工具调用直接触发的 dispatch@， + # 或 inbox 扫描独立确认的 "main 派给了谁"（inbox@dispatch:*）。 + # 排除 inbox@report:*／inbox@handoff:*——那是子 agent 上报/移交， + # 不是 main 发起的新派发，计进来会把"派发次数"虚高（同一次真实 + # 派发经常先触发 dispatch@，随后又在 inbox 里被自己的上报回声一次）。 + is_dispatch = evidence.startswith("dispatch@") or evidence.startswith("inbox@dispatch:") + if is_dispatch: + totals[str(agent)] += 1 + rows = [{"agent": agent, "count": count} for agent, count in totals.items()] + rows.sort(key=lambda r: r["count"], reverse=True) + return rows + + +def build_auto_dispatch_stats(project_root: Path) -> dict[str, int]: + log_path = project_root / ".codebuddy" / "hooks" / "logs" / "auto-dispatch.log" + stats = {"auto_dispatch": 0, "fallback_to_main": 0, "passthrough": 0} + if not log_path.is_file(): + return stats + with log_path.open("r", encoding="utf-8") as fp: + for line in fp: + line = line.strip() + if not line.startswith("{"): + continue + try: + rec = json.loads(line) + except Exception: + continue + decision = str(rec.get("decision") or "") + if decision == "auto_dispatch": + stats["auto_dispatch"] += 1 + elif decision == "fallback_to_main": + stats["fallback_to_main"] += 1 + elif decision in {"passthrough"}: + stats["passthrough"] += 1 + return stats + + +def build_cost_by_task_slug(events: list[dict[str, Any]]) -> dict[str, float]: + """按 task_slug 汇总 usage/stop 事件的 cost_usd——workflow-state.json 自己不知道成本， + 这是唯一能把 devflow 运行和真实花费对上的地方（需要 emitter 已经把 task_slug 挂到事件上， + 见 core/runtime.py 的 devflow 接入）。""" + totals: dict[str, float] = defaultdict(float) + for rec in events: + if rec.get("event") not in {"usage", "stop"}: + continue + task_slug = rec.get("task_slug") + cost = rec.get("cost_usd") + if task_slug and isinstance(cost, (int, float)): + totals[str(task_slug)] += float(cost) + return dict(totals) + + +def build_devflow_runs(project_root: Path, cost_by_slug: dict[str, float] | None = None) -> list[dict[str, Any]]: + artifacts_root = project_root / "artifacts" + runs: list[dict[str, Any]] = [] + cost_by_slug = cost_by_slug or {} + if not artifacts_root.is_dir(): + return runs + for child in sorted(artifacts_root.iterdir()): + state_path = child / "workflow-state.json" + raw = dv.read_workflow_state(str(state_path)) + if not isinstance(raw, dict) or not isinstance(raw.get("stages"), dict): + continue + snap = dv.stage_snapshot(raw) + size_class = str(snap.get("size_class") or "medium") + # medium/large 的 workflow-state.json 也会带着 SOLO key（模板固定写入， + # 状态停在 "pending" 或 "skipped"，从未被真正执行）——不能只看 key 存不存在， + # 必须看 SOLO 阶段是不是真的跑过，否则每个 medium/large 任务都会被误判成 + # solo 路径，阶段视图漏掉 TASK-01/02/03/CODE-REVIEW/TASK-04。 + solo_info = snap["stages"].get("SOLO") + solo_status = str(solo_info.get("status") or "") if isinstance(solo_info, dict) else "" + solo_actually_ran = solo_status not in {"", "pending", "skipped"} + is_solo = size_class == "small" or (size_class not in {"small", "medium", "large"} and solo_actually_ran) + if snap.get("schema_version") == "2.0": + order = PORTABLE_SOLO_ORDER if is_solo else PORTABLE_ORDER + else: + order = CLASSIC_SOLO_ORDER if is_solo else CLASSIC_ORDER + stages_out = [] + total_cost = cost_by_slug.get(child.name, 0.0) + run_start = None + run_end = None + overall = "completed" + for key in order: + info = snap["stages"].get(key) + if info is None: + # PHASE-0 在两套 schema 里都不会出现在 stages{} 里（它是隐式完成的： + # workflow-state.json 一旦存在，就说明 Phase 0 已经跑完了）， + # 不能用"没有条目"直接兜底成 pending，那样会把已完成的阶段显示错。 + info = {"status": "completed", "retry_count": 0, "executor": None} if key == "PHASE-0" \ + else {"status": "pending", "retry_count": 0, "executor": None} + label, name = STAGE_META.get(key, (key[:4], key)) + status = str(info.get("status") or "pending") + retry_count = int(info.get("retry_count") or 0) + raw_stage = raw.get("stages", {}).get(key) if isinstance(raw.get("stages"), dict) else {} + duration = 0 + if isinstance(raw_stage, dict): + started = raw_stage.get("started_at") + completed = raw_stage.get("completed_at") + if isinstance(started, str) and isinstance(completed, str): + try: + from datetime import datetime + t0 = datetime.fromisoformat(started.replace("Z", "+00:00")) + t1 = datetime.fromisoformat(completed.replace("Z", "+00:00")) + duration = max(0, int((t1 - t0).total_seconds())) + if run_start is None or t0 < run_start: + run_start = t0 + if run_end is None or t1 > run_end: + run_end = t1 + except Exception: + duration = 0 + if status == "failed": + overall = "paused" + elif status == "in_progress" and overall != "paused": + overall = "running" + stages_out.append({ + "key": key, "label": label, "name": name, + "exec": info.get("executor") or "-", "status": status, + "retry_count": retry_count, "duration": duration, + }) + # 按真实起止时间跨度算总时长（而不是逐阶段 duration 相加）——阶段之间可能 + # 有重叠（比如 TASK-05 的 started_at 早于 SOLO 的 completed_at），相加会 + # 把重叠部分重复计入，虚高于源数据本身反映的运行时长。 + total_duration = int((run_end - run_start).total_seconds()) if run_start and run_end else 0 + runs.append({ + "slug": child.name, "size": size_class, "stages": stages_out, + "cost": round(total_cost, 4), "duration": total_duration, "overall": overall, + }) + runs.sort(key=lambda r: r["slug"], reverse=True) + return runs + + +def resolve_input_paths( + skill_root: Path, + metrics_path: str | None = None, + state_path: str | None = None, +) -> tuple[Path, Path]: + """解析 metrics.ndjson 与 .state.json 的实际读取路径。 + + 任一参数为 None 时回退到 ``/logs/`` 下的默认路径，保证未传参时 + 行为与旧版本完全一致（向后兼容，不会破坏现有看板数据源）。 + + 非 None 时按用户给定路径（支持 ``~`` 展开）取绝对路径，便于看板从非默认 + 位置（如其它会话/项目的日志目录）聚合数据。 + """ + default_metrics = skill_root / "logs" / "metrics.ndjson" + default_state = skill_root / "logs" / ".state.json" + if metrics_path: + metrics = Path(metrics_path).expanduser().absolute() + else: + metrics = default_metrics + if state_path: + state = Path(state_path).expanduser().absolute() + else: + state = default_state + return metrics, state + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--project-root", default=".", help="devflow 项目根目录（含 .codebuddy/ 和 artifacts/）") + parser.add_argument("--out", default=None, help="输出路径，默认 /scripts/dashboard/dashboard-data.json") + parser.add_argument("--metrics-path", default=None, help="覆盖 metrics.ndjson 的读取路径（默认 /logs/metrics.ndjson，支持 ~ 展开）") + parser.add_argument("--state-path", default=None, help="覆盖 .state.json 的读取路径（默认 /logs/.state.json，支持 ~ 展开）") + parser.add_argument("--top-slow", type=int, default=None, help="额外在终端打印耗时最长的 N 次 tool 事件（工具名 + 耗时 ms），默认不打印；只打印不写入 dashboard-data.json") + args = parser.parse_args() + + project_root = Path(args.project_root).expanduser().resolve() + skill_root = SCRIPTS_DIR.parent + log_path, state_path = resolve_input_paths(skill_root, args.metrics_path, args.state_path) + + # 价格表带 lru_cache：本脚本可能长期驻留（配合文件监听器），单测也会同进程多次 + # 调 main()，每次都先清一次缓存，保证覆盖文件的改动当场生效。 + if em is not None: + try: + em.clear_price_cache() + except Exception: + pass + + events = read_ndjson(log_path) + state = safe_load_json(state_path) or {} + if not isinstance(state, dict): + state = {} + + daily, sessions = build_daily_and_sessions(events) + model_costs = build_model_costs(events) + unpriced = build_unpriced_models(events) + failures = build_failures(events) + skills, never_used = build_skills(state) + dispatch = build_dispatch(state) + auto_dispatch_stats = build_auto_dispatch_stats(project_root) + cost_by_slug = build_cost_by_task_slug(events) + devflow_runs = build_devflow_runs(project_root, cost_by_slug) + + from datetime import datetime, timezone + out_data = { + "generated_at": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "source": { + "metrics_ndjson": str(log_path), "state_json": str(state_path), + "project_root": str(project_root), "event_count": len(events), + }, + "daily": daily, "sessions": sessions, "modelCosts": model_costs, "unpricedModels": unpriced, + "failures": failures, + "skills": skills, "neverUsed": never_used, "dispatch": dispatch, + "autoDispatchStats": auto_dispatch_stats, "devflowRuns": devflow_runs, + } + + out_path = Path(args.out).expanduser().resolve() if args.out else (SCRIPTS_DIR / "dashboard" / "dashboard-data.json") + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(json.dumps(out_data, ensure_ascii=False, indent=2), encoding="utf-8") + print(f"wrote {out_path} ({len(events)} events, {len(sessions)} sessions, {len(devflow_runs)} devflow runs)") + # top-slow 放在原输出之后：先保证不传参时的终端输出与 JSON 行为完全不变。 + top_slow_block = format_top_slow(build_top_slow(events, args.top_slow)) + if top_slow_block: + print(top_slow_block) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.cursor/skills/agent-observability/scripts/core/__init__.py b/.cursor/skills/agent-observability/scripts/core/__init__.py new file mode 100644 index 0000000..1d6ad5e --- /dev/null +++ b/.cursor/skills/agent-observability/scripts/core/__init__.py @@ -0,0 +1,8 @@ +"""agent-observability 的核心运行时模块集合。 + +这里放的是 hook 主流程真正依赖的内部实现： +- 采集与 transcript 解析 +- 状态读写与去重 +- skill/rule 扫描 +- AgentLens trace 镜像 +""" diff --git a/.cursor/skills/agent-observability/scripts/core/agent_identity.py b/.cursor/skills/agent-observability/scripts/core/agent_identity.py new file mode 100644 index 0000000..270ff90 --- /dev/null +++ b/.cursor/skills/agent-observability/scripts/core/agent_identity.py @@ -0,0 +1,587 @@ +"""Agent 身份解析层。 + +这一层负责统一解析“当前是谁在工作、又把任务派给了谁”： +- 从 tool 调用参数里提取 agent 线索 +- 从 team inbox / mailbox 中补偿推断 agent 身份 +- 把解析结果写回 session state，供后续日志归因复用 +""" +from __future__ import annotations + +import json +import os +import re +import time +from datetime import datetime, timezone +from functools import lru_cache +from pathlib import Path +from typing import Any, Optional + +from . import scanner, state as st + +AGENT_PATH_RE = re.compile(r"\.codebuddy/agents/([a-zA-Z0-9_\-]+)(?:\.md)?", re.I) +DISPATCH_TOOLS = {"Task", "task", "Agent", "DeferExecuteTool"} +MESSAGE_TOOLS = {"send_message", "SendMessage"} +TEAM_PREFIX = "multi-agents-devflow-" +ROLE_INSTANCE_RE = re.compile(r"^(?P[a-z0-9][a-z0-9\-]*?)-\d+$", re.I) +INITIAL_ASSIGNMENT_RE = re.compile( + r"Initial task assignment for (?P[a-z0-9][a-z0-9\-]*)", + re.I, +) +ROLE_DECLARATION_RE = re.compile( + r"角色[：:]\s*(?P[a-z0-9][a-z0-9\-]*)", + re.I, +) +# devflow TASK-03 并行 fan-out 时，developer 会以 Task(name="sub-developer-PT-01", ...) +# 派发多条并行轨道。轨道号只在 transcript 文件名（= Task 的 name 参数）里出现， +# 按 subagent_name 归一化后会被折成同一个 "developer" —— 这是有意的（用于按角色汇总）， +# 轨道号需要单独提取，见 pt_id_from_transcript_path。 +PT_TRACK_RE = re.compile(r"-((?:PT|pt)-\d+)$") + + +def normalize_role_name(name: str | None) -> Optional[str]: + """把角色名归一化成稳定标识，例如把实例名折叠回基础角色名。""" + if not isinstance(name, str): + return None + cand = name.strip().lower() + if not cand: + return None + if cand.endswith(".json"): + cand = cand[:-5] + if "@" in cand: + cand = cand.split("@", 1)[0] + if cand in {"team-lead", "main"}: + return "main" + match = ROLE_INSTANCE_RE.match(cand) + if match: + cand = match.group("base") + return cand or None + + +def resolve_teams_root() -> Path: + """解析 CodeBuddy team inbox 的根目录。""" + config_dir = (os.environ.get("CODEBUDDY_CONFIG_DIR") or "").strip() + if config_dir: + return Path(config_dir).expanduser() / "teams" + return Path.home() / ".codebuddy" / "teams" + + +def resolve_team_dir(cwd: str, sid: str, cached_team_dir: str | None = None) -> Path | None: + """根据 session id 和 cwd 找到当前会话对应的 team 目录。""" + if cached_team_dir: + cached = Path(cached_team_dir).expanduser() + if (cached / "config.json").is_file(): + cfg = _load_json(cached / "config.json") + if isinstance(cfg, dict) and sid and str(cfg.get("leadSessionId") or "") == sid: + return cached + + if not sid: + return None + + teams_root = resolve_teams_root() + if not teams_root.is_dir(): + return None + + best_dir: Path | None = None + best_created = -1 + for team_dir in teams_root.iterdir(): + if not team_dir.is_dir() or not team_dir.name.startswith(TEAM_PREFIX): + continue + config = _load_json(team_dir / "config.json") + if not isinstance(config, dict): + continue + if str(config.get("leadSessionId") or "") != sid: + continue + if not _config_matches_cwd(config, cwd): + continue + + created_at = _safe_int(config.get("createdAt")) + if created_at <= 0: + try: + created_at = int(team_dir.stat().st_mtime * 1000) + except Exception: + created_at = 0 + if created_at > best_created: + best_dir = team_dir + best_created = created_at + return best_dir + + +def read_recent_messages( + team_dir: Path, + offsets: dict[str, int] | None = None, + *, + max_per_mailbox: int = 30, +) -> tuple[list[dict[str, Any]], dict[str, int]]: + """按 mailbox 增量读取最近消息，并返回新的 offset 游标。""" + inbox_dir = team_dir / "inboxes" + if not inbox_dir.is_dir(): + return [], offsets or {} + + prev_offsets = offsets or {} + new_offsets: dict[str, int] = {} + messages: list[dict[str, Any]] = [] + + for inbox_file in sorted(inbox_dir.glob("*.json")): + raw = _load_json(inbox_file) + if not isinstance(raw, list): + raw = [] + + total = len(raw) + prev = _safe_int(prev_offsets.get(inbox_file.name)) + if prev < 0 or prev > total: + prev = 0 + + start = prev + if start == 0 and total > max_per_mailbox: + start = total - max_per_mailbox + + mailbox_name = inbox_file.stem + mailbox_role = normalize_role_name(mailbox_name) + for idx, item in enumerate(raw[start:], start=start): + norm = _normalize_message(item, mailbox_name, mailbox_role, idx) + if norm: + messages.append(norm) + + new_offsets[inbox_file.name] = total + + messages.sort(key=lambda x: (x.get("ts") or 0.0, x.get("mailbox_name") or "", x.get("index") or 0)) + return messages, new_offsets + + +def infer_identity_from_messages(messages: list[dict[str, Any]]) -> tuple[Optional[str], Optional[str], dict[str, Any]]: + """根据 inbox 消息流推断当前 agent 与派发目标。""" + current: Optional[str] = None + dispatched: Optional[str] = None + evidence: Optional[str] = None + + for msg in messages: + mailbox_name = str(msg.get("mailbox_name") or "") + mailbox_role = normalize_role_name(msg.get("mailbox_role")) + from_role = normalize_role_name(msg.get("from_role")) + payload = msg.get("payload") if isinstance(msg.get("payload"), dict) else {} + next_target = _extract_next_target(payload) + + if from_role == "main" and mailbox_role and mailbox_role != "main": + if not msg.get("is_shutdown"): + current = mailbox_role + dispatched = mailbox_role + evidence = f"dispatch:{mailbox_name}" + continue + + if mailbox_name == "team-lead" and from_role and from_role not in {"main", "system"}: + report_role = normalize_role_name( + payload.get("from_role") if isinstance(payload, dict) else None + ) or from_role + if next_target: + current = next_target + dispatched = next_target + evidence = f"handoff:{report_role}->{next_target}" + else: + current = report_role + evidence = f"report:{report_role}" + + meta: dict[str, Any] = {"messages_seen": len(messages)} + if evidence: + meta["evidence"] = evidence + return current, dispatched, meta + + +def safe_agent_from_provider(record: dict[str, Any]) -> str | None: + """当 transcript 自己带有 providerData 时，直接读取其中的 agent。""" + provider = record.get("providerData") + if not isinstance(provider, dict): + return None + agent = provider.get("agent") + if isinstance(agent, str) and agent.strip(): + return normalize_role_name(agent.strip()) + return None + + +def role_from_content_items(content: Any) -> str | None: + """从 transcript content 文本中提取更具体的业务角色名。""" + if not isinstance(content, list): + return None + for item in content: + if not isinstance(item, dict): + continue + text = item.get("text") + if not isinstance(text, str): + continue + for regex in (INITIAL_ASSIGNMENT_RE, ROLE_DECLARATION_RE): + match = regex.search(text) + if not match: + continue + role = normalize_role_name(match.group("role")) + if role: + return role + return None + + +@lru_cache(maxsize=256) +def role_for_subagent_transcript(transcript_path: str) -> str | None: + """从 subagent transcript 前几行推断更准确的角色名。""" + path = Path(transcript_path) + try: + with path.open("r", encoding="utf-8") as fp: + for _ in range(6): + line = fp.readline() + if not line: + break + try: + record = json.loads(line) + except Exception: + continue + + # 对 subagent transcript，优先信任务分配文本里的具体角色， + # 再回退到 providerData.agent，避免 general-purpose 覆盖业务角色。 + content_role = role_from_content_items(record.get("content")) + if content_role: + return content_role + + provider_agent = safe_agent_from_provider(record) + if provider_agent: + return provider_agent + except Exception: + return None + return None + + +def agent_for_transcript_path(transcript_path: str, fallback: str | None = None) -> str: + """把 transcript 路径映射回实际对应的 agent 身份。""" + parts = Path(transcript_path).parts + if "subagents" in parts: + return role_for_subagent_transcript(transcript_path) or Path(transcript_path).stem + return fallback or "main" + + +def pt_id_from_transcript_path(transcript_path: str) -> str | None: + """从并行 sub-developer 的 transcript 文件名里提取 PT 轨道号（如 "PT-01"）。 + + 按事件自己的 transcript_path 推断，而不是写一个共享的 session 级"当前 PT"指针—— + devflow 一次可以并行派发最多 6 条 sub-developer 轨道，它们在同一个 sid 下 + 真·并发运行，任何"当前是哪条轨道"的可变共享状态在并发场景下都是错的。 + """ + parts = Path(transcript_path).parts + if "subagents" not in parts: + return None + match = PT_TRACK_RE.search(Path(transcript_path).stem) + return match.group(1).upper() if match else None + + +def merge_agent_identity_from_inbox( + state_path: Path, + sid: str, + cwd: str, + active_agent: str | None, + dispatched: str | None, +) -> tuple[str, str | None]: + """在基于 tool 的推断之上，再叠加 mailbox/inbox 的证据。""" + + def _update(state: dict[str, Any]) -> tuple[str, str | None]: + sess = st.ensure_session(state, sid) + cached_team_dir = str(sess.get("_team_dir") or "").strip() or None + offsets = sess.get("_inbox_offsets") + if not isinstance(offsets, dict): + offsets = {} + + effective_agent = active_agent or str(sess.get("current_agent") or "main") + effective_dispatched = dispatched + team_dir = resolve_team_dir(cwd, sid, cached_team_dir) + if team_dir is None: + return effective_agent, effective_dispatched + + sess["_team_dir"] = str(team_dir) + messages, new_offsets = read_recent_messages(team_dir, offsets) + sess["_inbox_offsets"] = new_offsets + inferred_current, inferred_dispatched, meta = infer_identity_from_messages(messages) + if inferred_current: + effective_agent = inferred_current + if inferred_dispatched: + effective_dispatched = inferred_dispatched + if inferred_current or inferred_dispatched: + target_agent = inferred_current or inferred_dispatched + if target_agent: + sess["current_agent"] = target_agent + hist = sess.setdefault("agent_history", []) + hist.append({ + "ts": time.time(), + "agent": target_agent, + "evidence": f"inbox@{meta.get('evidence') or 'inbox'}", + }) + return effective_agent, effective_dispatched + + return st.update_state_locked(state_path, _update) + + +def resolve_active_agent_for_event( + *, + state_path: Path, + sid: str, + cwd: str, + data: dict[str, Any], +) -> tuple[str, str | None]: + """优先用 tool 线索、其次用 inbox 线索，解析事件对应的 agent。""" + tracker = AgentIdentityResolver() + active_agent, dispatched = tracker.track(state_path, sid, data) + if scanner.is_brainstorming_call(data): + + def _update(state_data: dict[str, Any]) -> None: + sess = st.ensure_session(state_data, sid) + sess["current_agent"] = "main" + + st.update_state_locked(state_path, _update) + return "main", dispatched + return merge_agent_identity_from_inbox( + state_path, + sid, + cwd, + active_agent, + dispatched, + ) + + +class AgentIdentityResolver: + """基于 tool 调用内容做一轮 agent 身份推断。""" + def __init__(self, known_agents: set[str] | None = None): + self.known_agents = known_agents or set() + + def extract_identity(self, data: dict) -> tuple[Optional[str], Optional[str]]: + tool = data.get("tool_name", "") + tool_input = data.get("tool_input") or {} + if not isinstance(tool_input, dict): + tool_input = {} + + current: Optional[str] = None + dispatched: Optional[str] = None + + if tool in MESSAGE_TOOLS: + extracted_current, extracted_dispatched = self._extract_from_message(tool_input) + if extracted_current and not current: + current = extracted_current + if extracted_dispatched and not dispatched: + dispatched = extracted_dispatched + + if tool in DISPATCH_TOOLS and not dispatched: + dispatched = self._extract_from_task(tool_input) + + if not current: + current = self._extract_from_path(data, tool_input) + + if self.known_agents: + if current and current not in self.known_agents: + current = None + if dispatched and dispatched not in self.known_agents: + dispatched = None + + return current, dispatched + + def track(self, state_path: Path, sid: str, data: dict) -> tuple[str, Optional[str]]: + tool = data.get("tool_name", "") + cur_agent, dispatched = self.extract_identity(data) + + def _update(state: dict[str, Any]) -> tuple[str, Optional[str]]: + sess = st.ensure_session(state, sid) + hist = sess.setdefault("agent_history", []) + prev_agent = sess.get("current_agent") or "main" + + if cur_agent and cur_agent != prev_agent: + sess["current_agent"] = cur_agent + hist.append({"ts": time.time(), "agent": cur_agent, "evidence": f"from_role@{tool}"}) + active_agent = cur_agent + else: + active_agent = prev_agent + + if dispatched and dispatched != active_agent: + dis = sess.setdefault("dispatched", {}) + if not isinstance(dis, dict): + dis = {} + sess["dispatched"] = dis + dis[dispatched] = dis.get(dispatched, 0) + 1 + sess["current_agent"] = dispatched + hist.append({"ts": time.time(), "agent": dispatched, "evidence": f"dispatch@{tool}<-{active_agent}"}) + + if len(hist) > 100: + sess["agent_history"] = hist[-100:] + + return active_agent, dispatched + + return st.update_state_locked(state_path, _update) + + def _extract_from_message(self, tool_input: dict) -> tuple[Optional[str], Optional[str]]: + current: Optional[str] = None + dispatched: Optional[str] = None + content = tool_input.get("content") + + if isinstance(content, str) and content.strip().startswith("{"): + try: + payload = json.loads(content) + current = self._extract_str(payload, "from_role") + next_target = payload.get("next_target") or {} + if isinstance(next_target, dict): + dispatched = ( + self._extract_str(next_target, "role_name") + or self._extract_str(next_target, "subagent_name") + ) + except Exception: + pass + elif isinstance(content, dict): + current = self._extract_str(content, "from_role") + next_target = content.get("next_target") or {} + if isinstance(next_target, dict): + dispatched = ( + self._extract_str(next_target, "role_name") + or self._extract_str(next_target, "subagent_name") + ) + + if not dispatched: + recipient = tool_input.get("recipient") + if isinstance(recipient, str): + normalized = recipient.strip().lower() + if normalized and normalized != "main": + dispatched = normalized + + return current, dispatched + + def _extract_from_task(self, tool_input: dict) -> Optional[str]: + sub = ( + # Claude Code 的 Agent 工具（非 CodeBuddy 原生 Task/team_create）用的是 + # subagent_type 字段，不是 subagent_name——不认这个字段会导致 dispatch + # 统计漏掉所有走 Agent 工具派发的场景。 + tool_input.get("subagent_type") + or tool_input.get("subagent_name") + or tool_input.get("name") + or tool_input.get("agent") + or tool_input.get("role") + ) + if isinstance(sub, str) and sub.strip(): + candidate = sub.strip().lower() + if "/" in candidate: + candidate = candidate.rsplit("/", 1)[1].replace(".md", "") + return candidate + return None + + def _extract_from_path(self, data: dict, tool_input: dict) -> Optional[str]: + text = json.dumps(tool_input, ensure_ascii=False) + " " + (data.get("cwd") or "") + match = AGENT_PATH_RE.search(text) + if match: + return match.group(1).lower() + return None + + @staticmethod + def _extract_str(payload: dict, key: str) -> Optional[str]: + value = payload.get(key) + if isinstance(value, str) and value.strip(): + return value.strip().lower() + return None + + +def _config_matches_cwd(config: dict[str, Any], cwd: str) -> bool: + """判断 team 配置中的成员 cwd 是否覆盖当前工作目录。""" + members = config.get("members") + if not isinstance(members, list): + return False + for member in members: + if not isinstance(member, dict): + continue + member_cwd = member.get("cwd") + if isinstance(member_cwd, str) and _paths_equal(member_cwd, cwd): + return True + return False + + +def _paths_equal(a: str, b: str) -> bool: + """按归一化绝对路径语义比较两个路径是否相等。""" + try: + return Path(a).expanduser().resolve() == Path(b).expanduser().resolve() + except Exception: + return os.path.abspath(os.path.expanduser(a)) == os.path.abspath(os.path.expanduser(b)) + + +def _normalize_message(item: Any, mailbox_name: str, mailbox_role: str | None, index: int) -> dict[str, Any] | None: + """把原始 inbox 消息归一化成统一可推断的结构。""" + if not isinstance(item, dict): + return None + + text = item.get("text") + payload = _coerce_payload(text) + summary = item.get("summary") if isinstance(item.get("summary"), str) else "" + from_raw = item.get("from") if isinstance(item.get("from"), str) else None + from_role = normalize_role_name((payload.get("from") if isinstance(payload, dict) else None) or from_raw) + ts = _parse_timestamp(item.get("timestamp")) + + return { + "mailbox_name": mailbox_name, + "mailbox_role": mailbox_role, + "from": from_raw, + "from_role": from_role, + "text": text if isinstance(text, str) else "", + "summary": summary, + "payload": payload, + "timestamp": item.get("timestamp"), + "ts": ts, + "index": index, + "is_shutdown": _is_shutdown_message(payload, summary, text), + } + + +def _extract_next_target(payload: dict[str, Any] | None) -> Optional[str]: + """从 payload 中提取下一跳要派发给的角色。""" + if not isinstance(payload, dict): + return None + next_target = payload.get("next_target") or {} + if not isinstance(next_target, dict): + return None + return normalize_role_name(next_target.get("role_name") or next_target.get("subagent_name")) + + +def _coerce_payload(text: Any) -> dict[str, Any] | None: + """把文本内容尽量解析成 JSON 字典。""" + if isinstance(text, dict): + return text + if not isinstance(text, str): + return None + stripped = text.strip() + if not stripped.startswith("{"): + return None + try: + data = json.loads(stripped) + if isinstance(data, dict): + return data + except Exception: + return None + return None + + +def _is_shutdown_message(payload: dict[str, Any] | None, summary: str, text: Any) -> bool: + """识别一条 inbox 消息是否表示 agent 关闭或退出。""" + if isinstance(payload, dict) and str(payload.get("type") or "").lower() == "shutdown_request": + return True + summary_low = summary.lower() if isinstance(summary, str) else "" + text_low = text.lower() if isinstance(text, str) else "" + return "shutdown" in summary_low or "shutdown request" in text_low + + +def _parse_timestamp(value: Any) -> float: + """把消息时间字段解析成 Unix 时间戳。""" + if not isinstance(value, str) or not value.strip(): + return datetime.now(timezone.utc).timestamp() + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp() + except Exception: + return datetime.now(timezone.utc).timestamp() + + +def _load_json(path: Path) -> Any: + """安全读取 JSON 文件，失败时返回 ``None``。""" + try: + return json.loads(path.read_text("utf-8")) + except Exception: + return None + + +def _safe_int(value: Any) -> int: + """把任意值尽量转成整数，失败时返回 0。""" + try: + return int(value or 0) + except Exception: + return 0 diff --git a/.cursor/skills/agent-observability/scripts/core/agentlens/__init__.py b/.cursor/skills/agent-observability/scripts/core/agentlens/__init__.py new file mode 100644 index 0000000..2fea21d --- /dev/null +++ b/.cursor/skills/agent-observability/scripts/core/agentlens/__init__.py @@ -0,0 +1,9 @@ +"""可选的 AgentLens sink，用于实时镜像 trace。""" +from .runtime import emit_post_step, emit_session_start, emit_session_stop, emit_turn_start + +__all__ = [ + "emit_post_step", + "emit_session_start", + "emit_session_stop", + "emit_turn_start", +] diff --git a/.cursor/skills/agent-observability/scripts/core/agentlens/bootstrap.py b/.cursor/skills/agent-observability/scripts/core/agentlens/bootstrap.py new file mode 100644 index 0000000..d6fc970 --- /dev/null +++ b/.cursor/skills/agent-observability/scripts/core/agentlens/bootstrap.py @@ -0,0 +1,229 @@ +from __future__ import annotations +"""AgentLens 启动辅助层。 + +这一层负责“读入与初始化”： +- 解析项目配置与环境变量 +- 延迟加载可选的 zhiyan 运行时 +- 缓存初始化结果，避免重复 init +- 处理调试落盘与失败状态写回 +""" + +import getpass +import json +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from .. import state as st + +DEFAULT_APP_NAME = "skillhub.codebuddy-hooks" +DEFAULT_BUSINESS_SCENARIO = "codebuddy-hook" + +_RUNTIME: dict[str, Any] | None = None +_INIT_SIGNATURE: tuple[str, str, str] | None = None + + +@dataclass(frozen=True) +class AgentLensConfig: + """从环境变量与 env 文件归一化得到的 AgentLens 配置。""" + endpoint: str | None + api_key: str | None + app_name: str + business_scenario: str + user: str + + @property + def enabled(self) -> bool: + return bool(self.endpoint and self.api_key) + + +def project_root(cwd: str | None = None) -> Path: + """解析用于查找 `.env` 的项目根目录。""" + if cwd: + return Path(cwd).expanduser().resolve() + env_cwd = str(os.environ.get("CODEBUDDY_PROJECT_DIR") or "").strip() + if env_cwd: + return Path(env_cwd).expanduser().resolve() + return Path.cwd().resolve() + + +def read_env_file(path: Path) -> dict[str, str]: + """以轻量方式解析 shell 风格的 env 文件，不做 source。""" + values: dict[str, str] = {} + if not path.is_file(): + return values + try: + lines = path.read_text("utf-8").splitlines() + except Exception: + return values + + for raw_line in lines: + line = raw_line.strip() + if not line or line.startswith("#"): + continue + if line.startswith("export "): + line = line[7:].strip() + if "=" not in line: + continue + key, value = line.split("=", 1) + key = key.strip() + value = value.strip() + if not key: + continue + if value and value[0] == value[-1] and value[0] in {"'", '"'}: + value = value[1:-1] + values[key] = value + return values + + +def load_config(project_root_path: Path) -> AgentLensConfig: + """按优先级加载配置：进程环境变量 > `.env.local` > `.env`。""" + env_file = read_env_file(project_root_path / ".env") + env_local_file = read_env_file(project_root_path / ".env.local") + + def _value(name: str, default: str | None = None) -> str | None: + env_value = os.environ.get(name) + if env_value is not None and str(env_value).strip(): + return str(env_value).strip() + if name in env_local_file and str(env_local_file[name]).strip(): + return str(env_local_file[name]).strip() + if name in env_file and str(env_file[name]).strip(): + return str(env_file[name]).strip() + return default + + user = _value("ZHIYANLLM_USER") + if not user: + user = ( + str(os.environ.get("USER") or "").strip() + or str(os.environ.get("USERNAME") or "").strip() + or getpass.getuser() + ) + + return AgentLensConfig( + endpoint=_value("ZHIYANLLM_API_ENDPOINT"), + api_key=_value("ZHIYANLLM_API_KEY"), + app_name=_value("ZHIYANLLM_APP_NAME", DEFAULT_APP_NAME) or DEFAULT_APP_NAME, + business_scenario=_value("ZHIYANLLM_BUSINESS_SCENARIO", DEFAULT_BUSINESS_SCENARIO) + or DEFAULT_BUSINESS_SCENARIO, + user=user or "unknown", + ) + + +def load_runtime() -> dict[str, Any] | None: + """延迟导入可选的 zhiyan 运行时组件。 + + 即使本地没有安装 AgentLens 依赖，hook 主链路也必须继续工作， + 所以这里返回 ``None``，而不是把导入异常抛出去。 + """ + global _RUNTIME + if _RUNTIME is not None: + return _RUNTIME + try: + from zhiyanllm import Zhiyanllm + from zhiyanllm.opentelemetry.instrumentation.semconv_ai import ( + SpanAttributes as ZhiyanSpanAttributes, + ZhiyanllmSpanKindValues, + ) + from zhiyanllm.tracing.context_manager import get_tracer + from zhiyanllm.tracing.manual import track_llm_call, track_task_server_call + from zhiyanllm.tracing.tracing import TracerWrapper + from opentelemetry.context import attach as otel_attach, detach as otel_detach + except Exception: + _RUNTIME = None + return None + + _RUNTIME = { + "Zhiyanllm": Zhiyanllm, + "track_llm_call": track_llm_call, + "track_task_server_call": track_task_server_call, + "TracerWrapper": TracerWrapper, + "get_tracer": get_tracer, + "ZhiyanSpanAttributes": ZhiyanSpanAttributes, + "ZhiyanllmSpanKindValues": ZhiyanllmSpanKindValues, + "otel_attach": otel_attach, + "otel_detach": otel_detach, + } + return _RUNTIME + + +def ensure_initialized(config: AgentLensConfig) -> dict[str, Any] | None: + """仅当有效配置签名发生变化时才重新初始化 zhiyan。""" + global _INIT_SIGNATURE + if not config.enabled: + return None + runtime = load_runtime() + if runtime is None: + return None + signature = (str(config.endpoint), str(config.api_key), str(config.app_name)) + if _INIT_SIGNATURE == signature: + return runtime + runtime["Zhiyanllm"].init( + app_name=str(config.app_name), + api_endpoint=str(config.endpoint), + api_key=str(config.api_key), + disable_batch=True, + ) + _INIT_SIGNATURE = signature + return runtime + + +def project_root_from_state_path(state_path: Path) -> Path: + """尽力从 `logs/.state.json` 反推出项目根目录。""" + try: + current = state_path.resolve().parent + for parent in [current, *current.parents]: + if parent.name == ".codebuddy": + return parent.parent + return state_path.resolve().parents[4] + except Exception: + return Path.cwd().resolve() + + +def debug_enabled(state_path: Path) -> bool: + """判断是否要把 span 调试信息镜像写入 `agentlens-push-debug.ndjson`。""" + root = project_root_from_state_path(state_path) + env_file = read_env_file(root / ".env") + env_local_file = read_env_file(root / ".env.local") + raw = ( + os.environ.get("AGENTLENS_PUSH_DEBUG") + or env_local_file.get("AGENTLENS_PUSH_DEBUG") + or env_file.get("AGENTLENS_PUSH_DEBUG") + or "" + ) + return str(raw).strip().lower() in {"1", "true", "yes", "on"} + + +def debug_write_span( + state_path: Path | None, + sid: str | None, + payload: dict[str, Any], + *, + sanitizer, +) -> None: + """在开启 AgentLens 调试模式时追加写入清洗后的调试记录。""" + if state_path is None or not sid: + return + try: + if not debug_enabled(state_path): + return + debug_path = state_path.parent / "agentlens-push-debug.ndjson" + record = {"sid": sid, **sanitizer(payload)} + with debug_path.open("a", encoding="utf-8") as fp: + fp.write(json.dumps(record, ensure_ascii=False, sort_keys=True) + "\n") + except Exception: + return + + +def set_failure(state_path: Path, sid: str, message: str) -> None: + """记录最近一次 AgentLens 失败，但不打断 hook 主流程。""" + st.update_agentlens_session( + state_path, + sid, + {"enabled": False, "last_error": str(message)}, + ) + + +def get_session_context(state_path: Path, sid: str) -> dict[str, Any]: + """从共享状态里读取 AgentLens sidecar 的 session payload。""" + return st.load_agentlens_session(state_path, sid) diff --git a/.cursor/skills/agent-observability/scripts/core/agentlens/normalize.py b/.cursor/skills/agent-observability/scripts/core/agentlens/normalize.py new file mode 100644 index 0000000..2b730f0 --- /dev/null +++ b/.cursor/skills/agent-observability/scripts/core/agentlens/normalize.py @@ -0,0 +1,323 @@ +from __future__ import annotations +"""AgentLens 负载清洗层。 + +这一层负责“清洗与整形”： +- 在发送或调试前把任意 payload 转成安全的 JSON 结构 +- 把本地 token 结构归一化成类似 OpenAI 的 usage 结构 +- 从 transcript 中重建轻量的 message / tool-call 上下文 +""" + +import json +import os +from pathlib import Path +from typing import Any + +from .. import state as st +from .bootstrap import AgentLensConfig + + +def sanitize_payload(value: Any) -> Any: + """把任意 Python 值转换成适合调试/打点的 JSON 安全结构。""" + try: + return json.loads(json.dumps(value, ensure_ascii=False, default=str)) + except Exception: + return str(value) + + +def association_properties(config: AgentLensConfig, sid: str, state_path: Path | None = None) -> dict[str, str]: + """构造 trace 级关联属性，并允许按 turn 覆盖业务场景。""" + scenario = config.business_scenario + if state_path: + try: + session_ctx = st.load_agentlens_session(state_path, sid) + turn_scenario = session_ctx.get("_turn_business_scenario") + if isinstance(turn_scenario, str) and turn_scenario.strip(): + scenario = turn_scenario.strip() + except Exception: + pass + return { + "session_id": sid, + "business_scenario": scenario, + "user": config.user, + } + + +def to_openai_usage(tokens: dict[str, Any] | None) -> dict[str, int]: + """把本地 token 结构转换成 zhiyan 期望的 usage 形状。""" + if not isinstance(tokens, dict): + return {} + + def _int(v: Any) -> int: + try: + return int(v or 0) + except Exception: + return 0 + + prompt = _int(tokens.get("input")) + completion = _int(tokens.get("output")) + if prompt <= 0 and completion <= 0: + return {} + return {"prompt_tokens": prompt, "completion_tokens": completion, "total_tokens": prompt + completion} + + +def infer_vendor(model: str | None) -> str: + """根据模型名推断一个粗粒度的 provider/vendor 标签。""" + text = str(model or "").strip().lower() + if not text: + return "unknown" + if "gpt" in text or "openai" in text: + return "openai" + if "claude" in text or "anthropic" in text: + return "anthropic" + if "gemini" in text or "google" in text: + return "google" + if text.startswith("hy") or "hunyuan" in text: + return "tencent" + return text.split("/", 1)[0].split("-", 1)[0] or "unknown" + + +def extract_message_text(content: Any) -> str: + """把嵌套的 transcript message 内容压平成纯文本。""" + if content is None: + return "" + if isinstance(content, str): + return content + if isinstance(content, list): + parts: list[str] = [] + for item in content: + if isinstance(item, dict): + txt = item.get("text") + if isinstance(txt, str): + parts.append(txt) + elif item.get("type") == "tool_use": + parts.append(f"[tool_use {item.get('name', '')}]") + elif item.get("type") == "tool_result": + parts.append(extract_message_text(item.get("content"))) + else: + parts.append(str(item)) + return "\n".join(p for p in parts if p) + if isinstance(content, dict): + return extract_message_text(content.get("text") or content.get("content")) + return str(content) + + +def find_message_id_in_record(obj: Any) -> str | None: + """在 transcript 记录里递归查找稳定的 message 标识。""" + message_id_keys = ("messageId", "responseId", "requestId") + if isinstance(obj, dict): + provider = obj.get("providerData") + if isinstance(provider, dict): + for key in message_id_keys: + value = provider.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + extra = obj.get("extra") + if isinstance(extra, dict): + for key in message_id_keys: + value = extra.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + for key in message_id_keys: + value = obj.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + for value in obj.values(): + found = find_message_id_in_record(value) + if found: + return found + elif isinstance(obj, list): + for value in obj: + found = find_message_id_in_record(value) + if found: + return found + return None + + +def collect_tool_calls_for_message(transcript_path: str, message_id: str) -> list[dict[str, Any]]: + """为指定 assistant message 重建它挂载的 tool-call 摘要。""" + if not transcript_path or not message_id or not os.path.isfile(transcript_path): + return [] + try: + with open(transcript_path, "rb") as fp: + blob = fp.read() + except Exception: + return [] + + tool_calls: list[dict[str, Any]] = [] + for raw_line in blob.splitlines(keepends=True): + if not raw_line.lstrip().startswith(b"{"): + continue + try: + rec = json.loads(raw_line.decode("utf-8", errors="ignore")) + except Exception: + continue + if not isinstance(rec, dict): + continue + if str(rec.get("type") or "").strip().lower() != "function_call": + continue + rec_mid = find_message_id_in_record(rec) + if rec_mid != message_id: + continue + call_id = str(rec.get("callId") or "").strip() + name = str(rec.get("name") or "").strip() + arguments = rec.get("arguments") + entry: dict[str, Any] = {"id": call_id, "name": name} + if arguments is not None: + entry["arguments"] = arguments if isinstance(arguments, str) else json.dumps(arguments, ensure_ascii=False) + tool_calls.append(entry) + return tool_calls + + +def find_tool_usage_event(tool_event: dict[str, Any], usage_events: list[dict[str, Any]]) -> dict[str, Any] | None: + """从 usage 列表里挑出最可能属于当前 tool 事件的那一条。""" + tool_name = str(tool_event.get("tool") or "") + transcript_path = str(tool_event.get("transcript_path") or "") + agent = str(tool_event.get("agent") or "main") + for usage_event in usage_events: + if not isinstance(usage_event, dict): + continue + if str(usage_event.get("agent") or "main") != agent: + continue + if transcript_path and str(usage_event.get("transcript_path") or "") != transcript_path: + continue + if tool_name and str(usage_event.get("tool") or "") not in {tool_name, "model_request"}: + continue + if str(usage_event.get("message_id") or "").strip(): + return usage_event + for usage_event in usage_events: + if not isinstance(usage_event, dict): + continue + if str(usage_event.get("agent") or "main") != agent: + continue + if transcript_path and str(usage_event.get("transcript_path") or "") != transcript_path: + continue + return usage_event + return None + + +def build_llm_io_from_transcript( + *, + transcript_path: str, + source_offset: int, + tokens: dict[str, Any] | None, + model: str | None, + start_offset: int = 0, + current_message_id: str | None = None, +) -> tuple[dict[str, Any], dict[str, Any], str]: + """从 transcript 历史中构造紧凑版的 LLM 输入/输出负载。""" + input_data: dict[str, Any] = {"model": str(model or "unknown"), "messages": []} + output_data: dict[str, Any] = {"choices": [], "usage": to_openai_usage(tokens)} + if not transcript_path or not os.path.isfile(transcript_path): + return input_data, output_data, "" + try: + with open(transcript_path, "rb") as fp: + fp.seek(0) + blob = fp.read(source_offset) if source_offset else fp.read() + except Exception: + return input_data, output_data, "" + + messages: list[dict[str, Any]] = [] + system_prompts: list[str] = [] + cursor = 0 + last_asst_text: str | None = None + + def _truncate(text: str, limit: int = 4000) -> str: + if len(text) <= limit: + return text + return text[:limit] + "...(truncated)" + + for raw_line in blob.splitlines(keepends=True): + cursor += len(raw_line) + if not raw_line.lstrip().startswith(b"{"): + continue + try: + rec = json.loads(raw_line.decode("utf-8", errors="ignore")) + except Exception: + continue + if not isinstance(rec, dict): + continue + + rec_type = str(rec.get("type") or "").strip().lower() + is_incremental = cursor > start_offset + + # 重建 prompt 上下文时要跳过“当前正在发出的 assistant message”， + # 否则同一段回复会同时出现在 input 和 output 两边。 + if current_message_id and is_incremental: + rec_mid = find_message_id_in_record(rec) + if rec_mid == current_message_id: + if rec_type == "message" and rec.get("role") == "assistant": + text = extract_message_text(rec.get("content")) + if text: + last_asst_text = text + continue + + if rec_type == "message": + role = rec.get("role") + text = extract_message_text(rec.get("content")) + if not text: + continue + if role == "system": + system_prompts.append(text) + elif is_incremental: + if role == "user": + messages.append({"role": "user", "content": _truncate(text, 8000)}) + last_asst_text = None + elif role == "assistant": + messages.append({"role": "assistant", "content": _truncate(text, 4000)}) + last_asst_text = text + elif is_incremental: + # 把 function call / result 记录转换成 Chat Completions 风格的 + # assistant/tool message，方便 tracing UI 按对话链路展示。 + if rec_type == "function_call": + name = str(rec.get("name") or "").strip() + call_id = str(rec.get("callId") or "").strip() + arguments = rec.get("arguments", "") + if isinstance(arguments, dict): + arguments = json.dumps(arguments, ensure_ascii=False) + tool_call: dict[str, Any] = {"type": "function", "function": {"name": name}} + if name and arguments: + tool_call["function"]["arguments"] = _truncate(str(arguments), 2000) + if call_id: + tool_call["id"] = call_id + if ( + messages + and messages[-1].get("role") == "assistant" + and "tool_calls" not in messages[-1] + and messages[-1].get("content") + ): + messages[-1]["tool_calls"] = [tool_call] + else: + msg: dict[str, Any] = {"role": "assistant", "content": ""} + msg["tool_calls"] = [tool_call] + messages.append(msg) + elif rec_type == "function_call_result": + call_id = str(rec.get("callId") or "").strip() + provider = rec.get("providerData", {}) + result_content = "" + if isinstance(provider, dict): + tool_result = provider.get("toolResult", {}) + if isinstance(tool_result, dict): + result_content = extract_message_text(tool_result.get("content")) + if not result_content: + output = rec.get("output") + result_content = str(output.get("text", "")) if isinstance(output, dict) else "" + tool_msg: dict[str, Any] = {"role": "tool", "content": _truncate(result_content, 2000)} + if call_id: + tool_msg["tool_call_id"] = call_id + messages.append(tool_msg) + + final_messages: list[dict[str, Any]] = [] + for prompt in system_prompts[-2:]: + if prompt: + final_messages.append({"role": "system", "content": _truncate(prompt, 2000)}) + final_messages.extend(messages) + input_data["messages"] = final_messages + + if last_asst_text is not None: + output_data["choices"] = [{ + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": last_asst_text[:8000]}, + }] + + return input_data, output_data, "" diff --git a/.cursor/skills/agent-observability/scripts/core/agentlens/runtime.py b/.cursor/skills/agent-observability/scripts/core/agentlens/runtime.py new file mode 100644 index 0000000..321f45a --- /dev/null +++ b/.cursor/skills/agent-observability/scripts/core/agentlens/runtime.py @@ -0,0 +1,761 @@ +"""AgentLens 运行时编排层。 + +这一层保留对外暴露的 4 个入口： +- `emit_session_start` +- `emit_turn_start` +- `emit_post_step` +- `emit_session_stop` + +它本身不负责底层清洗或 carrier 生成，而是把调用编排到 +`bootstrap / normalize / tracing` 三层上。 +""" +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from opentelemetry import trace as _otel_trace + +from .. import state as st +from .bootstrap import ( + debug_write_span as _debug_write_span, + ensure_initialized as _ensure_initialized, + get_session_context as _get_session_context, + load_config, + project_root as _project_root, + set_failure as _set_failure, +) +from .normalize import ( + association_properties as _association_properties, + build_llm_io_from_transcript as _build_llm_io_from_transcript, + find_tool_usage_event as _find_tool_usage_event, + infer_vendor as _infer_vendor, + sanitize_payload as _sanitize_payload, + to_openai_usage as _to_openai_usage, +) +from .tracing import ( + annotate as _annotate, + emit_assistant_span as _emit_assistant_span, + emit_tool_span as _emit_tool_span, + generate_subagent_carrier as _generate_subagent_carrier, + generate_turn_carrier as _generate_turn_carrier, + resolve_agent_carrier as _resolve_agent_carrier, + resolve_step_carrier as _resolve_step_carrier, + set_agent_aggregate_on_span as _set_agent_aggregate_on_span, + span_context_ids as _span_context_ids, + traceparent_parts as _traceparent_parts, +) + + +def emit_session_start( + *, + state_path: Path, + sid: str, + cwd: str, + agent: str = "main", +) -> None: + """初始化 v2 版本的 `_agentlens` session payload。 + + 在 turn-centric 模型下，这里**不再**生成 trace_id。 + 它只负责记录 session 级元数据（如 `enabled`、`session_id`、`app_name`）， + 并清理上一次运行遗留的 carrier / error 状态。 + 真正的新 trace 会在第一次 `UserPromptSubmit` 时由 `emit_turn_start` 打开。 + """ + _ = agent + config = load_config(_project_root(cwd)) + # 注意：这里不要清空 current_turn / turn_history。 + # 同一个 sid 下 SessionStart 可能多次触发（例如 subagent 启动、IDE 刷新、 + # workspace 切换）。如果这里清空，会把进行中的 turn 擦掉，后续 PostToolUse + # 会全部看到 no_active_turn，最终把这一轮观测链路打断。 + base_updates: dict[str, Any] = { + "session_id": sid, + "app_name": config.app_name, + } + if not config.enabled: + base_updates["enabled"] = False + st.update_agentlens_session( + state_path, + sid, + base_updates, + clear_keys=["carrier", "last_error"], + ) + return + + try: + runtime = _ensure_initialized(config) + if runtime is None: + _set_failure(state_path, sid, "zhiyanllm runtime unavailable") + return + base_updates["enabled"] = True + st.update_agentlens_session( + state_path, + sid, + base_updates, + clear_keys=["carrier", "last_error"], + ) + except Exception as exc: + _set_failure(state_path, sid, str(exc)) + + +def emit_turn_start( + *, + state_path: Path, + sid: str, + cwd: str, + turn_id: str, + prompt_meta: dict[str, Any] | None = None, + business_scenario: str | None = None, +) -> None: + """打开一个新的 turn，生成新的 trace_id 并写入 `_agentlens.current_turn`。 + + 这是当前实现里**唯一**允许生成新 trace_id 的地方。 + `emit_post_step` / `emit_session_stop` 都不会重新造 carrier； + 找不到时只会优雅降级，不会私自开新链路。 + """ + _ = prompt_meta + config = load_config(_project_root(cwd)) + if not config.enabled: + return + + try: + runtime = _ensure_initialized(config) + if runtime is None: + _set_failure(state_path, sid, "zhiyanllm runtime unavailable") + return + # 先把 per-turn business scenario 写进 state， + # 这样 `_generate_turn_carrier -> _association_properties` + # 才能在生成 trace 时把这次 turn 的业务场景烘焙进去。 + if business_scenario: + st.update_agentlens_session( + state_path, + sid, + {"enabled": True, "session_id": sid, "app_name": config.app_name, "_turn_business_scenario": business_scenario}, + clear_keys=["last_error"], + ) + else: + st.update_agentlens_session( + state_path, + sid, + {"enabled": True, "session_id": sid, "app_name": config.app_name}, + clear_keys=["last_error", "_turn_business_scenario"], + ) + carrier = _generate_turn_carrier(runtime, config=config, sid=sid, turn_id=turn_id, state_path=state_path) + st.begin_turn(state_path, sid, turn_id, carrier) + except Exception as exc: + _set_failure(state_path, sid, str(exc)) + + +def emit_post_step( + *, + state_path: Path, + sid: str, + tool_event: dict[str, Any] | None, + usage_events: list[dict[str, Any]], +) -> None: + """发出扁平化的 LLM span，只保留最小 TASK 包装层来传播 carrier 上下文。 + + 面向 turn 的 v2 约束： + - 活跃的 ``current_turn.carrier`` 是唯一的 trace 上下文来源。 + - 如果不存在 ``current_turn``（例如 UserPromptSubmit hook 还没触发）， + 这里只会记录 ``last_error`` 并返回，不会私自生成新的 trace_id。 + - subagent（``active_agent != "main"``）会拿到当前 turn 下的 + ``invoke_agent`` 子 span，后续 chat span 都挂到这个子 span 下， + 以保持因果链路连续。 + """ + if not isinstance(tool_event, dict) and not usage_events: + return + + cwd_hint = "" + if isinstance(tool_event, dict): + cwd_hint = str(tool_event.get("cwd") or "") + config = load_config(_project_root(cwd_hint)) + if not config.enabled: + return + + session_ctx = _get_session_context(state_path, sid) + # 快路径：尊重 SessionStart 阶段已经写入的 enabled 标记。 + if session_ctx.get("enabled") is False: + return + + current_turn = session_ctx.get("current_turn") + if not isinstance(current_turn, dict): + _set_failure(state_path, sid, "no_active_turn") + return + parent_carrier = current_turn.get("carrier") + if not isinstance(parent_carrier, dict) or not parent_carrier.get("traceparent"): + _set_failure(state_path, sid, "no_active_turn_carrier") + return + + try: + runtime = _ensure_initialized(config) + if runtime is None: + _set_failure(state_path, sid, "zhiyanllm runtime unavailable") + return + + tool_name = str((tool_event or {}).get("tool") or "unknown_tool") + active_agent = str((tool_event or {}).get("agent") or "main") + transcript_path = str((tool_event or {}).get("transcript_path") or "") + skills = (tool_event or {}).get("skill") or [] + rules = (tool_event or {}).get("rule") or [] + + # 解析这次事件真正要用的 carrier。 + # main agent 直接使用 turn 根 carrier；subagent 则使用按角色拆分的 + # 子 span carrier，并在当前 turn 下做幂等注册。 + carrier = dict(parent_carrier) + if active_agent and active_agent != "main": + existing = None + subs = current_turn.get("subagent_spans") + if isinstance(subs, dict): + rec = subs.get(active_agent) + if isinstance(rec, dict) and isinstance(rec.get("carrier"), dict): + existing = rec["carrier"] + if isinstance(existing, dict) and existing.get("traceparent"): + carrier = dict(existing) + else: + # 原子化的 get-or-create：只有确认 subagent span 不存在时， + # 才会在写锁内部调用 carrier_factory。 + # 这样可以避免并发 hook 进程之间的 TOCTOU 竞争，产生孤儿 + # invoke_agent.TASK span。 + def _factory() -> dict[str, str]: + return _generate_subagent_carrier( + runtime, parent_carrier=parent_carrier, role=active_agent + ) + + registered = st.upsert_subagent_span_atomic( + state_path, sid, active_agent, carrier_factory=_factory + ) + if isinstance(registered, dict) and registered.get("traceparent"): + carrier = dict(registered) + runtime["Zhiyanllm"].set_association_properties(_association_properties(config, sid, state_path)) + # 静默挂载 carrier 上下文（不额外创建 TASK span），让内部 span + # 继承正确的 trace_id / parent_id。 + _ctx = runtime["TracerWrapper"].extract_context(carrier) + _token = runtime["otel_attach"](_ctx) + try: + preferred_tool_message_id = str((tool_event or {}).get("message_id") or "").strip() or None + _assistant_emitted_mids: set[str] = set() + pre_bumped_steps: set[tuple[str, str]] = set() + committed_steps: set[tuple[str, str]] = set() + + if isinstance(tool_event, dict): + if preferred_tool_message_id and st.get_step_span_carrier( + state_path, sid, active_agent, preferred_tool_message_id + ) is None: + pre_bumped_steps.add((active_agent, preferred_tool_message_id)) + for usage_event in usage_events: + event_agent = str(usage_event.get("agent") or active_agent or "main") + event_message_id = str(usage_event.get("message_id") or "").strip() or None + if event_message_id: + step_key = (event_agent, event_message_id) + if ( + step_key not in pre_bumped_steps + and st.get_step_span_carrier(state_path, sid, event_agent, event_message_id) is None + ): + pre_bumped_steps.add(step_key) + if not usage_events and isinstance(tool_event, dict): + tool_parent_carrier = _resolve_agent_carrier( + state_path=state_path, + sid=sid, + runtime=runtime, + parent_carrier=carrier, + agent=active_agent, + ) + if preferred_tool_message_id: + step_carrier = _resolve_step_carrier( + state_path=state_path, + sid=sid, + runtime=runtime, + parent_carrier=tool_parent_carrier, + agent=active_agent, + message_id=preferred_tool_message_id, + transcript_path=transcript_path, + ) + if isinstance(step_carrier, dict) and step_carrier.get("traceparent"): + tool_parent_carrier = dict(step_carrier) + _emit_tool_span( + runtime, + carrier=tool_parent_carrier, + tool_name=tool_name, + active_agent=active_agent, + skills=skills, + rules=rules, + duration_ms=tool_event.get("ms") if isinstance(tool_event.get("ms"), (int, float)) else None, + tool_details=tool_event.get("tool_details") if isinstance(tool_event.get("tool_details"), dict) else None, + message_id=preferred_tool_message_id, + step_grouping="message_id" if preferred_tool_message_id else "fallback", + state_path=state_path, + sid=sid, + ) + st.bump_agent_aggregate( + state_path, + sid, + active_agent, + tool_event=tool_event, + ) + if ( + preferred_tool_message_id + and (active_agent, preferred_tool_message_id) in pre_bumped_steps + and (active_agent, preferred_tool_message_id) not in committed_steps + ): + st.bump_agent_aggregate( + state_path, + sid, + active_agent, + step_created=True, + ) + committed_steps.add((active_agent, preferred_tool_message_id)) + st.update_agentlens_session(state_path, sid, {"enabled": True}, clear_keys=["last_error"]) + return + + matched_usage = _find_tool_usage_event(tool_event, usage_events) if isinstance(tool_event, dict) else None + tool_message_id = preferred_tool_message_id or str((matched_usage or {}).get("message_id") or "").strip() or None + if isinstance(tool_event, dict): + # 先构造 assistant 预览，再打开 agent/step 父 span。 + # 这样即便 transcript 重建失败，也不会留下空的分组 TASK。 + _assistant_preview_content: str | None = None + _assistant_preview_transcript = transcript_path + _assistant_preview_model = "" + if tool_message_id: + for _ue in usage_events: + if str(_ue.get("message_id") or "").strip() != tool_message_id: + continue + _ue_transcript = str(_ue.get("transcript_path") or transcript_path) + _ue_offset = int(_ue.get("source_offset") or 0) + _, _llm_out_pre, _ = _build_llm_io_from_transcript( + transcript_path=_ue_transcript, + source_offset=_ue_offset, + tokens=_ue.get("tokens"), + model=str(_ue.get("model") or ""), + ) + _pre_content = "" + if _llm_out_pre.get("choices"): + _pre_msg = _llm_out_pre["choices"][0].get("message") or {} + _pre_content = str(_pre_msg.get("content") or "") + if _pre_content and _pre_content.strip(): + _assistant_preview_content = _pre_content + _assistant_preview_transcript = _ue_transcript + _assistant_preview_model = str(_ue.get("model") or "") + break + + tool_parent_carrier = _resolve_agent_carrier( + state_path=state_path, + sid=sid, + runtime=runtime, + parent_carrier=carrier, + agent=active_agent, + ) + if tool_message_id: + step_carrier = _resolve_step_carrier( + state_path=state_path, + sid=sid, + runtime=runtime, + parent_carrier=tool_parent_carrier, + agent=active_agent, + message_id=tool_message_id, + transcript_path=transcript_path, + ) + if isinstance(step_carrier, dict) and step_carrier.get("traceparent"): + tool_parent_carrier = dict(step_carrier) + if _assistant_preview_content and tool_message_id: + _emit_assistant_span( + runtime, + carrier=tool_parent_carrier, + agent=active_agent, + message_id=tool_message_id, + assistant_text=_assistant_preview_content, + model=_assistant_preview_model, + transcript_path=_assistant_preview_transcript, + state_path=state_path, + sid=sid, + ) + _assistant_emitted_mids.add(tool_message_id) + + _emit_tool_span( + runtime, + carrier=tool_parent_carrier, + tool_name=tool_name, + active_agent=active_agent, + skills=skills, + rules=rules, + duration_ms=tool_event.get("ms") if isinstance(tool_event.get("ms"), (int, float)) else None, + tool_details=tool_event.get("tool_details") if isinstance(tool_event.get("tool_details"), dict) else None, + message_id=tool_message_id, + step_grouping="message_id" if tool_message_id else "fallback", + state_path=state_path, + sid=sid, + ) + st.bump_agent_aggregate( + state_path, + sid, + active_agent, + tool_event=tool_event, + ) + if tool_message_id and (active_agent, tool_message_id) in pre_bumped_steps: + step_key = (active_agent, tool_message_id) + if step_key not in committed_steps: + st.bump_agent_aggregate( + state_path, + sid, + active_agent, + step_created=True, + ) + committed_steps.add(step_key) + + _prev_llm_offsets: dict[str, int] = {} + for usage_event in usage_events: + model = str(usage_event.get("model") or "") + ev_transcript = str(usage_event.get("transcript_path") or transcript_path) + ev_offset = int(usage_event.get("source_offset") or 0) + raw_tokens = usage_event.get("tokens") or {} + event_message_id = str(usage_event.get("message_id") or "").strip() or None + event_agent = str(usage_event.get("agent") or active_agent or "main") + start_offset = _prev_llm_offsets.get(ev_transcript) or st.get_last_llm_offset( + st.load_state(state_path), sid, ev_transcript + ) + llm_input, llm_output, _turn_title = _build_llm_io_from_transcript( + transcript_path=ev_transcript, + source_offset=ev_offset, + start_offset=int(start_offset or 0), + current_message_id=event_message_id, + tokens=raw_tokens, + model=model or None, + ) + _prev_llm_offsets[ev_transcript] = ev_offset + # 原子写回 state，供跨进程增量跟踪复用 + def _update_llm_offset(state: dict[str, Any], _ev=ev_transcript, _off=ev_offset) -> None: + st.set_last_llm_offset(state, sid, _off, _ev) + st.update_state_locked(state_path, _update_llm_offset) + # --- 把 skill/rule/agent 作为独立消息标签注入 input_data --- + _meta_messages: list[dict[str, str]] = [] + _ev_agent = str(usage_event.get("agent") or active_agent) + if _ev_agent and _ev_agent != "main": + _meta_messages.append({"role": "agent", "content": _ev_agent}) + if skills: + _meta_messages.append({"role": "skill", "content": ", ".join(skills)}) + if rules: + _meta_messages.append({"role": "rule", "content": ", ".join(rules)}) + if _meta_messages: + llm_input.setdefault("messages", []) + for msg in reversed(_meta_messages): + llm_input["messages"].insert(0, msg) + # --- 构造 usage --- + llm_output["usage"] = _to_openai_usage(raw_tokens) + # --- 只附带成本细节来构造 output 内容 --- + _cost_info = { + "input_tokens": raw_tokens.get("input"), + "output_tokens": raw_tokens.get("output"), + "cache_read": raw_tokens.get("cache_read"), + "cache_creation": raw_tokens.get("cache_creation"), + "total_tokens": raw_tokens.get("total"), + "cost_usd": usage_event.get("cost_usd"), + } + _cost_info = {k: v for k, v in _cost_info.items() if v is not None} + _existing_content = "" + if llm_output.get("choices"): + _msg = llm_output["choices"][0].get("message") or {} + _existing_content = str(_msg.get("content") or "") + _output_content = _existing_content + if _cost_info: + _cost_line = " | ".join(f"{k}: {v}" for k, v in _cost_info.items()) + _output_content = f"[{_cost_line}]\n{_existing_content}" if _existing_content else f"[{_cost_line}]" + llm_output["choices"] = [{ + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": _output_content}, + }] + event_parent_carrier = carrier + if event_message_id: + event_parent_carrier = _resolve_agent_carrier( + state_path=state_path, + sid=sid, + runtime=runtime, + parent_carrier=carrier, + agent=event_agent, + ) + step_carrier = _resolve_step_carrier( + state_path=state_path, + sid=sid, + runtime=runtime, + parent_carrier=event_parent_carrier, + agent=event_agent, + message_id=event_message_id, + transcript_path=ev_transcript, + ) + if isinstance(step_carrier, dict) and step_carrier.get("traceparent"): + event_parent_carrier = dict(step_carrier) + ev_ctx = runtime["TracerWrapper"].extract_context(event_parent_carrier) + ev_token = runtime["otel_attach"](ev_ctx) + # 先累计聚合值，再发 LLM span，这样 span 上带的是最新累计属性。 + st.bump_agent_aggregate( + state_path, + sid, + event_agent, + usage_event=usage_event, + ) + _current_agg = st.get_subagent_aggregate(state_path, sid, event_agent) + try: + with runtime["track_llm_call"](_infer_vendor(model), "model_request") as llm_span: + # track_llm_call 已经设置了 gen_ai.span.kind/system/operation.name。 + # 下方 _annotate 会通过 handle_llm_response 设置 gen_ai.usage.*。 + # LLMSpan 包装层没有 set_attribute，所以这里改用原生 span 写 ID。 + _native_span = _otel_trace.get_current_span() + _set_agent_aggregate_on_span(_native_span, _current_agg) + span_ids = _span_context_ids(_native_span) + parent_parts = _traceparent_parts(event_parent_carrier) + _debug_write_span( + state_path, + sid, + { + "event": "span_emit", + "span_kind": "llm", + "span_name": "model_request", + "model": model, + "message_id": str(event_message_id or ""), + "step_grouping": "message_id" if event_message_id else "fallback", + "agent": event_agent, + "trace_id": span_ids.get("trace_id", parent_parts.get("trace_id", "")), + "span_id": span_ids.get("span_id", ""), + "parent_span_id": parent_parts.get("span_id", ""), + "parent_traceparent": parent_parts.get("traceparent", ""), + "transcript_path": ev_transcript, + "source_offset": ev_offset, + }, + sanitizer=_sanitize_payload, + ) + _annotate( + runtime, + span=llm_span, + input_data=llm_input, + output_data=llm_output, + tags={ + "cost_usd": str(usage_event.get("cost_usd") or "-"), + "input_tokens": str(raw_tokens.get("input", "-")), + "output_tokens": str(raw_tokens.get("output", "-")), + "cache_read": str(raw_tokens.get("cache_read", "-")), + "cache_creation": str(raw_tokens.get("cache_creation", "-")), + "total_tokens": str(raw_tokens.get("total", "-")), + "message_id": str(event_message_id or ""), + "step_grouping": "message_id" if event_message_id else "fallback", + }, + ) + finally: + runtime["otel_detach"](ev_token) + if event_message_id and (event_agent, event_message_id) in pre_bumped_steps: + step_key = (event_agent, event_message_id) + if step_key not in committed_steps: + st.bump_agent_aggregate( + state_path, + sid, + event_agent, + step_created=True, + ) + committed_steps.add(step_key) + if _existing_content and _existing_content.strip() and event_message_id not in _assistant_emitted_mids: + _emit_assistant_span( + runtime, + carrier=event_parent_carrier, + agent=event_agent, + message_id=event_message_id, + assistant_text=_existing_content, + model=model, + transcript_path=ev_transcript, + state_path=state_path, + sid=sid, + ) + finally: + runtime["otel_detach"](_token) + st.update_agentlens_session(state_path, sid, {"enabled": True}, clear_keys=["last_error"]) + except Exception as exc: + _set_failure(state_path, sid, str(exc)) + + +def emit_session_stop( + *, + state_path: Path, + sid: str, + cwd: str, + active_agent: str, + transcript_path: str, + stop_events: list[dict[str, Any]], +) -> None: + """在 session stop 阶段发出扁平化 LLM span，只保留最小 TASK 包装层传播上下文。""" + config = load_config(_project_root(cwd)) + if not config.enabled: + return + + session_ctx = _get_session_context(state_path, sid) + if session_ctx.get("enabled") is False: + return + + current_turn = session_ctx.get("current_turn") + if not isinstance(current_turn, dict): + _set_failure(state_path, sid, "no_active_turn") + return + carrier = current_turn.get("carrier") + if not isinstance(carrier, dict) or not carrier.get("traceparent"): + _set_failure(state_path, sid, "no_active_turn_carrier") + return + + try: + runtime = _ensure_initialized(config) + if runtime is None: + _set_failure(state_path, sid, "zhiyanllm runtime unavailable") + return + + runtime["Zhiyanllm"].set_association_properties(_association_properties(config, sid, state_path)) + if not stop_events: + return + # 静默挂载 carrier 上下文（不额外创建 TASK span）。 + _ctx = runtime["TracerWrapper"].extract_context(carrier) + _token = runtime["otel_attach"](_ctx) + try: + _prev_stop_offsets: dict[str, int] = {} + for stop_event in stop_events: + model = str(stop_event.get("model") or "") + ev_transcript = str(stop_event.get("transcript_path") or transcript_path) + ev_offset = int(stop_event.get("source_offset") or 0) + raw_tokens = stop_event.get("tokens") or {} + event_message_id = str(stop_event.get("message_id") or "").strip() or None + event_agent = str(stop_event.get("agent") or active_agent or "main") + start_offset = _prev_stop_offsets.get(ev_transcript) or st.get_last_llm_offset( + st.load_state(state_path), sid, ev_transcript + ) + llm_input, llm_output, _turn_title = _build_llm_io_from_transcript( + transcript_path=ev_transcript, + source_offset=ev_offset, + start_offset=int(start_offset or 0), + current_message_id=event_message_id, + tokens=raw_tokens, + model=model or None, + ) + _prev_stop_offsets[ev_transcript] = ev_offset + # 原子写回 state，供跨进程增量跟踪复用 + def _update_llm_offset_stop(state: dict[str, Any], _ev=ev_transcript, _off=ev_offset) -> None: + st.set_last_llm_offset(state, sid, _off, _ev) + st.update_state_locked(state_path, _update_llm_offset_stop) + # --- 把 agent 作为独立消息标签注入 input_data --- + _ev_agent = str(stop_event.get("agent") or active_agent) + if _ev_agent and _ev_agent != "main": + llm_input.setdefault("messages", []).insert(0, {"role": "agent", "content": _ev_agent}) + # --- 构造 usage --- + llm_output["usage"] = _to_openai_usage(raw_tokens) + # --- 构造带上下文和成本细节的 output 内容 --- + _cost_info = { + "input_tokens": raw_tokens.get("input"), + "output_tokens": raw_tokens.get("output"), + "cache_read": raw_tokens.get("cache_read"), + "cache_creation": raw_tokens.get("cache_creation"), + "total_tokens": raw_tokens.get("total"), + "cost_usd": stop_event.get("cost_usd"), + "cost_session_usd": stop_event.get("cost_session_usd"), + } + _cost_info = {k: v for k, v in _cost_info.items() if v is not None} + _existing_content = "" + if llm_output.get("choices"): + _msg = llm_output["choices"][0].get("message") or {} + _existing_content = str(_msg.get("content") or "") + _output_content = _existing_content + if _cost_info: + _cost_line = " | ".join(f"{k}: {v}" for k, v in _cost_info.items()) + _output_content = f"[{_cost_line}]\n{_existing_content}" if _existing_content else f"[{_cost_line}]" + llm_output["choices"] = [{ + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": _output_content}, + }] + event_parent_carrier = carrier + # 先解析 agent carrier，让 step 挂在 agent 的 react_agent span 下， + # 而不是直接挂在 turn 根 span 下。 + if event_agent and event_agent != "main": + agent_carrier = _resolve_agent_carrier( + state_path=state_path, + sid=sid, + runtime=runtime, + parent_carrier=carrier, + agent=event_agent, + ) + if isinstance(agent_carrier, dict) and agent_carrier.get("traceparent"): + event_parent_carrier = dict(agent_carrier) + if event_message_id: + step_carrier = _resolve_step_carrier( + state_path=state_path, + sid=sid, + runtime=runtime, + parent_carrier=event_parent_carrier, + agent=event_agent, + message_id=event_message_id, + transcript_path=ev_transcript, + ) + if isinstance(step_carrier, dict) and step_carrier.get("traceparent"): + event_parent_carrier = dict(step_carrier) + ev_ctx = runtime["TracerWrapper"].extract_context(event_parent_carrier) + ev_token = runtime["otel_attach"](ev_ctx) + # 这里不要再 bump，emit_post_step 已经累计过这条 usage_event。 + # SessionStop 会为同一次调用补发一个重复的 LLM span，再累加就会双算。 + # 因此这里只读取当前聚合值。 + _current_agg = st.get_subagent_aggregate(state_path, sid, event_agent) + try: + with runtime["track_llm_call"](_infer_vendor(model), "model_request") as llm_span: + # track_llm_call 已经设置了 gen_ai.span.kind/system/operation.name。 + # 下方 _annotate 会通过 handle_llm_response 设置 gen_ai.usage.*。 + # LLMSpan 包装层没有 set_attribute，所以这里改用原生 span 写 ID。 + _native_span = _otel_trace.get_current_span() + _set_agent_aggregate_on_span(_native_span, _current_agg) + span_ids = _span_context_ids(_native_span) + parent_parts = _traceparent_parts(event_parent_carrier) + _debug_write_span( + state_path, + sid, + { + "event": "span_emit", + "span_kind": "llm", + "span_name": "model_request", + "model": model, + "message_id": str(event_message_id or ""), + "step_grouping": "message_id" if event_message_id else "fallback", + "agent": event_agent, + "trace_id": span_ids.get("trace_id", parent_parts.get("trace_id", "")), + "span_id": span_ids.get("span_id", ""), + "parent_span_id": parent_parts.get("span_id", ""), + "parent_traceparent": parent_parts.get("traceparent", ""), + "transcript_path": ev_transcript, + "source_offset": ev_offset, + }, + sanitizer=_sanitize_payload, + ) + _annotate( + runtime, + span=llm_span, + input_data=llm_input, + output_data=llm_output, + tags={ + "cost_usd": str(stop_event.get("cost_usd") or "-"), + "cost_session_usd": str(stop_event.get("cost_session_usd") or "-"), + "input_tokens": str(raw_tokens.get("input", "-")), + "output_tokens": str(raw_tokens.get("output", "-")), + "cache_read": str(raw_tokens.get("cache_read", "-")), + "cache_creation": str(raw_tokens.get("cache_creation", "-")), + "total_tokens": str(raw_tokens.get("total", "-")), + "message_id": str(event_message_id or ""), + "step_grouping": "message_id" if event_message_id else "fallback", + }, + ) + finally: + runtime["otel_detach"](ev_token) + if _existing_content and _existing_content.strip(): + _emit_assistant_span( + runtime, + carrier=event_parent_carrier, + agent=event_agent, + message_id=event_message_id, + assistant_text=_existing_content, + model=model, + transcript_path=ev_transcript, + state_path=state_path, + sid=sid, + ) + finally: + runtime["otel_detach"](_token) + st.update_agentlens_session(state_path, sid, {"enabled": True}, clear_keys=["last_error"]) + except Exception as exc: + _set_failure(state_path, sid, str(exc)) diff --git a/.cursor/skills/agent-observability/scripts/core/agentlens/tracing.py b/.cursor/skills/agent-observability/scripts/core/agentlens/tracing.py new file mode 100644 index 0000000..afb678a --- /dev/null +++ b/.cursor/skills/agent-observability/scripts/core/agentlens/tracing.py @@ -0,0 +1,504 @@ +from __future__ import annotations +"""AgentLens trace 拓扑层。 + +这一层负责“trace 结构本身”： +- 创建或派生 carrier +- 生成 agent / step / assistant / tool span +- 把后续聚合需要的 span 元数据写回 state +""" + +import json +import secrets +from pathlib import Path +from typing import Any + +from .. import state as st +from .bootstrap import AgentLensConfig, debug_write_span +from .normalize import association_properties, collect_tool_calls_for_message, sanitize_payload + + +def annotate(runtime: dict[str, Any], *, span: Any | None = None, input_data: Any = None, output_data: Any = None, tags: dict[str, Any] | None = None) -> None: + """在调用 zhiyan annotate 前，先统一清洗 payload。""" + _ = span + runtime["Zhiyanllm"].annotate( + input_data=sanitize_payload(input_data), + output_data=sanitize_payload(output_data), + tags=sanitize_payload(tags) if tags else None, + ) + + +def traceparent_parts(carrier: dict[str, str] | None) -> dict[str, str]: + """把 W3C `traceparent` 拆成调试日志里常用的几个字段。""" + traceparent = str((carrier or {}).get("traceparent") or "").strip() + parts = traceparent.split("-") + if len(parts) >= 4: + return { + "traceparent": traceparent, + "trace_id": parts[1], + "span_id": parts[2], + "flags": parts[3], + } + return {"traceparent": traceparent} + + +def span_context_ids(span: Any) -> dict[str, str]: + """从 OTel span 对象中提取十六进制的 trace/span id。""" + try: + ctx = span.get_span_context() + return { + "trace_id": f"{int(ctx.trace_id):032x}", + "span_id": f"{int(ctx.span_id):016x}", + } + except Exception: + return {} + + +def ensure_traceparent(carrier: dict[str, str]) -> None: + """当上游没有成功注入时，补一个最小可用的 `traceparent`。""" + if str(carrier.get("traceparent") or "").strip(): + return + trace_id = secrets.token_hex(16) + parent_id = secrets.token_hex(8) + carrier["traceparent"] = f"00-{trace_id}-{parent_id}-01" + + +def carrier_from_span(parent_carrier: dict[str, str], span: Any) -> dict[str, str]: + """基于父 carrier 和当前 span 生成一个仍在同一 trace 上的子 carrier。""" + try: + span_ctx = span.get_span_context() + trace_id = f"{int(span_ctx.trace_id):032x}" + span_id = f"{int(span_ctx.span_id):016x}" + except Exception: + return {} + parent_tp = str((parent_carrier or {}).get("traceparent") or "").strip() + flags = "01" + if parent_tp: + parts = parent_tp.split("-") + if len(parts) >= 4 and parts[3]: + flags = parts[3] + child_carrier: dict[str, str] = dict(parent_carrier or {}) + child_carrier["traceparent"] = f"00-{trace_id}-{span_id}-{flags}" + return child_carrier + + +def carrier_from_task_span(parent_carrier: dict[str, str], task_span: Any) -> dict[str, str]: + """兼容包装过的 task span 和原始 OTel span 两种形态。""" + span = getattr(task_span, "_span", None) + if span is None and hasattr(task_span, "get_span_context"): + span = task_span + if span is None: + return {} + return carrier_from_span(parent_carrier, span) + + +def zhiyan_attr(runtime: dict[str, Any], name: str, fallback: str) -> str: + """防御式读取 zhiyan 语义约定常量。""" + return str(getattr(runtime.get("ZhiyanSpanAttributes"), name, fallback)) + + +def generate_turn_carrier( + runtime: dict[str, Any], + *, + config: AgentLensConfig, + sid: str, + turn_id: str, + state_path: Path | None = None, +) -> dict[str, str]: + """为一个用户 turn 生成根 carrier。""" + from opentelemetry import trace + + carrier: dict[str, str] = {} + span = trace.get_tracer(__name__).start_span(f"turn.{sid}.{turn_id}") + try: + runtime["Zhiyanllm"].set_association_properties(association_properties(config, sid, state_path)) + runtime["Zhiyanllm"].inject_context(carrier) + finally: + span.end() + ensure_traceparent(carrier) + return carrier + + +def generate_subagent_carrier(runtime: dict[str, Any], *, parent_carrier: dict[str, str], role: str) -> dict[str, str]: + """在当前 turn 下面创建 `invoke_agent` 子 carrier。""" + try: + with runtime["track_task_server_call"]("invoke_agent", carrier=parent_carrier) as task_span: + annotate(runtime, input_data={"agent": role}, output_data={"result": "dispatched"}, tags={"agent": role}) + child_carrier = carrier_from_task_span(parent_carrier, task_span) + if child_carrier: + return child_carrier + except Exception: + return {} + return {} + + +def generate_step_carrier( + runtime: dict[str, Any], + *, + parent_carrier: dict[str, str], + agent: str, + message_id: str, + transcript_path: str, +) -> dict[str, str]: + """创建用于归并同一条 message 工作的 synthetic step span。""" + try: + with runtime["track_task_server_call"]("react_step", carrier=parent_carrier) as task_span: + annotate( + runtime, + input_data={"agent": agent, "message_id": message_id}, + output_data={"result": "grouped"}, + tags={"message_id": message_id, "agent": agent, "transcript_path": transcript_path or ""}, + ) + child_carrier = carrier_from_task_span(parent_carrier, task_span) + if child_carrier: + return child_carrier + except Exception: + return {} + return {} + + +def generate_agent_carrier(runtime: dict[str, Any], *, parent_carrier: dict[str, str], agent: str) -> dict[str, str]: + """在当前 turn 下创建稳定的 per-agent 聚合 span。""" + try: + with runtime["track_task_server_call"]("react_agent", carrier=parent_carrier) as task_span: + span = getattr(task_span, "_span", None) + if span is not None and hasattr(span, "set_attribute"): + span.set_attribute("agent.name", agent) + annotate(runtime, input_data={"agent": agent}, output_data={"agent": agent}, tags={"agent": agent}) + child_carrier = carrier_from_task_span(parent_carrier, task_span) + if child_carrier: + return child_carrier + except Exception: + return {} + return {} + + +def resolve_agent_carrier( + *, + state_path: Path, + sid: str, + runtime: dict[str, Any], + parent_carrier: dict[str, str], + agent: str, +) -> dict[str, str]: + """从共享状态里获取或创建持久化的 agent 聚合 carrier。""" + normalized_agent = str(agent or "main").strip() or "main" + existing = st.get_agent_span_carrier(state_path, sid, normalized_agent) + if isinstance(existing, dict) and existing.get("traceparent"): + return dict(existing) + + def _factory() -> dict[str, str]: + return generate_agent_carrier(runtime, parent_carrier=parent_carrier, agent=normalized_agent) + + effective_carrier = st.upsert_agent_span_atomic(state_path, sid, normalized_agent, carrier_factory=_factory) + if not isinstance(effective_carrier, dict) or not effective_carrier.get("traceparent"): + return dict(parent_carrier) + agent_parts = traceparent_parts(effective_carrier) + parent_parts = traceparent_parts(parent_carrier) + debug_write_span( + state_path, + sid, + { + "event": "span_emit", + "span_kind": "agent", + "span_name": "react_agent", + "agent": normalized_agent, + "trace_id": agent_parts.get("trace_id", parent_parts.get("trace_id", "")), + "span_id": agent_parts.get("span_id", ""), + "parent_span_id": parent_parts.get("span_id", ""), + "parent_traceparent": parent_parts.get("traceparent", ""), + }, + sanitizer=sanitize_payload, + ) + return dict(effective_carrier) + + +def resolve_step_carrier( + *, + state_path: Path, + sid: str, + runtime: dict[str, Any], + parent_carrier: dict[str, str], + agent: str, + message_id: str | None, + transcript_path: str, +) -> dict[str, str] | None: + """获取或创建按 message 分组使用的 step carrier。""" + normalized_message_id = str(message_id or "").strip() + if not normalized_message_id: + return None + existing = st.get_step_span_carrier(state_path, sid, agent, normalized_message_id) + if isinstance(existing, dict) and existing.get("traceparent"): + return dict(existing) + + def _factory() -> dict[str, str]: + return generate_step_carrier( + runtime, + parent_carrier=parent_carrier, + agent=agent, + message_id=normalized_message_id, + transcript_path=transcript_path, + ) + + effective_carrier = st.upsert_step_span_atomic( + state_path, + sid, + agent, + normalized_message_id, + carrier_factory=_factory, + transcript_path=transcript_path, + ) + if not isinstance(effective_carrier, dict) or not effective_carrier.get("traceparent"): + return None + step_parts = traceparent_parts(effective_carrier) + parent_parts = traceparent_parts(parent_carrier) + debug_write_span( + state_path, + sid, + { + "event": "span_emit", + "span_kind": "step", + "span_name": "react_step", + "message_id": normalized_message_id, + "agent": agent, + "trace_id": step_parts.get("trace_id", parent_parts.get("trace_id", "")), + "span_id": step_parts.get("span_id", ""), + "parent_span_id": parent_parts.get("span_id", ""), + "parent_traceparent": parent_parts.get("traceparent", ""), + "transcript_path": transcript_path or "", + }, + sanitizer=sanitize_payload, + ) + return dict(effective_carrier) + + +def set_agent_aggregate_on_span(span: Any, aggregate: dict[str, Any] | None) -> None: + """把 agent 的累计指标投影成当前 span 的属性。""" + if span is None or not hasattr(span, "set_attribute") or not isinstance(aggregate, dict): + return + tokens = aggregate.get("tokens") or {} + if isinstance(tokens, dict): + for key in ("input", "output", "cache_read", "cache_creation", "total"): + if key in tokens: + span.set_attribute(f"agent.cumulative_tokens.{key}", int(tokens.get(key) or 0)) + for key in ("cost_usd", "tool_duration_ms", "llm_call_count", "tool_call_count", "step_count", "event_count"): + if key in aggregate: + try: + span.set_attribute(f"agent.cumulative.{key}", int(aggregate.get(key) or 0)) + except (TypeError, ValueError): + span.set_attribute(f"agent.cumulative.{key}", str(aggregate.get(key))) + + +def emit_assistant_span( + runtime: dict[str, Any], + *, + carrier: dict[str, str], + agent: str, + message_id: str | None, + assistant_text: str, + model: str | None = None, + transcript_path: str | None = None, + state_path: Path | None = None, + sid: str | None = None, +) -> None: + """发出一个轻量 assistant span，并按需补上 tool calls。""" + if not assistant_text or not assistant_text.strip(): + return + tool_calls: list[dict[str, Any]] = [] + if message_id and transcript_path: + tool_calls = collect_tool_calls_for_message(transcript_path, message_id) + output_data: dict[str, Any] = {"role": "assistant", "content": assistant_text[:8000]} + if tool_calls: + output_data["tool_calls"] = tool_calls + ctx = runtime["TracerWrapper"].extract_context(carrier) + token = runtime["otel_attach"](ctx) + try: + with runtime["get_tracer"]() as tracer: + with tracer.start_as_current_span(name="assistant_message") as span: + span.set_attribute(runtime["ZhiyanSpanAttributes"].LLM_SPAN_KIND, runtime["ZhiyanllmSpanKindValues"].LLM.value) + span_ids = span_context_ids(span) + parent_parts = traceparent_parts(carrier) + debug_write_span( + state_path, + sid, + { + "event": "span_emit", + "span_kind": "assistant", + "span_name": "assistant_message", + "message_id": str(message_id or ""), + "agent": agent, + "model": str(model or ""), + "tool_call_count": len(tool_calls), + "trace_id": span_ids.get("trace_id", parent_parts.get("trace_id", "")), + "span_id": span_ids.get("span_id", ""), + "parent_span_id": parent_parts.get("span_id", ""), + "parent_traceparent": parent_parts.get("traceparent", ""), + }, + sanitizer=sanitize_payload, + ) + annotate( + runtime, + input_data={"agent": agent, "message_id": message_id or "", "model": model or ""}, + output_data=output_data, + tags={"message_id": str(message_id or ""), "agent": agent}, + ) + finally: + runtime["otel_detach"](token) + + +def emit_tool_span( + runtime: dict[str, Any], + *, + carrier: dict[str, str], + tool_name: str, + active_agent: str, + skills: list[str], + rules: list[str], + duration_ms: int | None = None, + tool_details: dict[str, Any] | None = None, + message_id: str | None = None, + step_grouping: str = "fallback", + state_path: Path | None = None, + sid: str | None = None, +) -> None: + """发出 tool span，并以受控大小挂载 tool payload。""" + def _truncate(value: Any, limit: int = 1200) -> Any: + text = value if isinstance(value, str) else None + if text is None: + return value + if len(text) <= limit: + return text + return text[:limit] + "...(truncated)" + + def _maybe_parse_json_text(value: Any) -> Any: + if not isinstance(value, str): + return value + text = value.strip() + if not text or text[0] not in "{[": + return value + try: + return json.loads(text) + except Exception: + return value + + def _parse_bash_result_content(value: Any) -> dict[str, Any] | None: + if not isinstance(value, str) or "Command:" not in value: + return None + normalized_value = value.replace("\\n", "\n") + markers = [ + ("Command:", "command"), + ("Stdout:", "stdout"), + ("Stderr:", "stderr"), + ("Exit Code:", "exit_code"), + ("Signal:", "signal"), + ] + parsed: dict[str, Any] = {} + for idx, (marker, key) in enumerate(markers): + start = normalized_value.find(marker) + if start < 0: + continue + start += len(marker) + end = len(normalized_value) + for next_marker, _ in markers[idx + 1:]: + pos = normalized_value.find(next_marker, start) + if pos >= 0: + end = min(end, pos) + segment = normalized_value[start:end].strip() + if not segment: + continue + parsed[key] = _truncate(segment, 1200 if key in {"command", "stdout", "stderr"} else 200) + return parsed or None + + tool_input: dict[str, Any] = {"tool": tool_name, "agent": active_agent} + if skills: + tool_input["skill"] = ", ".join(skills) + if rules: + tool_input["rule"] = ", ".join(rules) + if message_id: + tool_input["message_id"] = message_id + details = dict(tool_details or {}) + call_id = str(details.get("call_id") or "").strip() or None + if call_id: + tool_input["call_id"] = call_id + arguments_display_text = details.get("arguments_display_text") + if arguments_display_text: + tool_input["arguments_display_text"] = _truncate(arguments_display_text, 600) + arguments = details.get("arguments") + if arguments: + parsed_arguments = _maybe_parse_json_text(arguments) + tool_input["arguments"] = _truncate(parsed_arguments, 1200) + if isinstance(parsed_arguments, dict): + if parsed_arguments.get("command"): + tool_input["command"] = _truncate(parsed_arguments.get("command"), 1200) + if parsed_arguments.get("description"): + tool_input["description"] = _truncate(parsed_arguments.get("description"), 400) + + tool_output: dict[str, Any] = {"result": "executed"} + result_content = details.get("result_content") + if result_content is not None: + structured_result = _parse_bash_result_content(result_content) + if structured_result: + tool_output["result"] = structured_result + else: + tool_output["result_content"] = _truncate(result_content, 1600) + output_text = details.get("output_text") + if output_text: + tool_output["output_text"] = _truncate(output_text, 1600) + raw_response = details.get("raw_response") + if isinstance(raw_response, dict): + tool_output["raw_response"] = { + "exitCode": raw_response.get("exitCode"), + "signal": raw_response.get("signal"), + "interrupted": raw_response.get("interrupted"), + "sandboxDenied": raw_response.get("sandboxDenied"), + "tool_error_code": raw_response.get("tool_error_code"), + } + + ctx = runtime["TracerWrapper"].extract_context(carrier) + token = runtime["otel_attach"](ctx) + try: + with runtime["get_tracer"]() as tracer: + with tracer.start_as_current_span(name=f"{tool_name}.TOOL") as span: + span_kind_value = runtime["ZhiyanllmSpanKindValues"].TOOL.value + span.set_attribute(zhiyan_attr(runtime, "LLM_SPAN_KIND", "gen_ai.span.kind"), span_kind_value) + span.set_attribute("gen_ai.span.kind", span_kind_value) + span.set_attribute(zhiyan_attr(runtime, "TOOL_NAME", "tool.name"), tool_name) + if isinstance(tool_details, dict): + desc = tool_details.get("description") or tool_details.get("tool_description") + if isinstance(desc, str) and desc.strip(): + span.set_attribute(zhiyan_attr(runtime, "TOOL_DESCRIPTION", "tool.description"), desc.strip()) + tool_params = tool_input.get("arguments") + if tool_params is not None: + if not isinstance(tool_params, str): + tool_params = json.dumps(tool_params, ensure_ascii=False, default=str) + span.set_attribute(zhiyan_attr(runtime, "TOOL_PARAMETERS", "tool.parameters"), tool_params) + if duration_ms is not None: + span.set_attribute("tool.duration_ms", int(duration_ms)) + span_ids = span_context_ids(span) + parent_parts = traceparent_parts(carrier) + debug_write_span( + state_path, + sid, + { + "event": "span_emit", + "span_kind": "tool", + "span_name": f"{tool_name}.TOOL", + "tool": tool_name, + "message_id": str(message_id or ""), + "call_id": call_id or "", + "step_grouping": step_grouping, + "agent": active_agent, + "trace_id": span_ids.get("trace_id", parent_parts.get("trace_id", "")), + "span_id": span_ids.get("span_id", ""), + "parent_span_id": parent_parts.get("span_id", ""), + "parent_traceparent": parent_parts.get("traceparent", ""), + }, + sanitizer=sanitize_payload, + ) + annotate( + runtime, + input_data=tool_input, + output_data=tool_output, + tags={"step_grouping": step_grouping, "message_id": str(message_id or "")}, + ) + finally: + runtime["otel_detach"](token) diff --git a/.cursor/skills/agent-observability/scripts/core/cls_sink.py b/.cursor/skills/agent-observability/scripts/core/cls_sink.py new file mode 100644 index 0000000..6525f6e --- /dev/null +++ b/.cursor/skills/agent-observability/scripts/core/cls_sink.py @@ -0,0 +1,470 @@ +from __future__ import annotations + +import hashlib +import hmac +import json +import os +import struct +import subprocess +import time +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from .agentlens import bootstrap + + + +@dataclass(frozen=True) +class CLSConfig: + enabled: bool + endpoint: str + topic_id: str + secret_id: str + secret_key: str + secret_token: str + service_name: str + timeout_seconds: int + helper_path: Path + sdk_entry_path: Path + + @property + def ready(self) -> bool: + return bool( + self.enabled + and self.endpoint + and self.topic_id + and self.secret_id + and self.secret_key + and self.helper_path.is_file() + and self.sdk_entry_path.is_file() + ) + + +def repo_root() -> Path: + return Path(__file__).resolve().parents[5] + + +def helper_path() -> Path: + return Path(__file__).with_name("cls_uploader.mjs") + + +def sdk_entry_path() -> Path: + return ( + repo_root() + / "cls-codebuddy" + / "tencentcloud-cls-sdk-codebuddy" + / "node_modules" + / "tencentcloud-cls-sdk-js" + / "dist" + / "index.js" + ) + + +def debug_log_path() -> Path: + return Path(__file__).resolve().parents[2] / "logs" / "cls-push-debug.ndjson" + + +def _value( + name: str, + *, + env_local: dict[str, str], + env_file: dict[str, str], + default: str = "", + aliases: tuple[str, ...] = (), +) -> str: + keys = (name, *aliases) + for key in keys: + raw = os.environ.get(key) + if raw is not None and str(raw).strip(): + return str(raw).strip() + for key in keys: + raw = env_local.get(key) + if raw is not None and str(raw).strip(): + return str(raw).strip() + for key in keys: + raw = env_file.get(key) + if raw is not None and str(raw).strip(): + return str(raw).strip() + return default + + +def _enabled_flag(*, env_local: dict[str, str], env_file: dict[str, str]) -> bool: + raw = _value( + "CLS_CODINGAGENT_ENABLED", + env_local=env_local, + env_file=env_file, + default="1", + ) + return str(raw).strip().lower() not in {"0", "false", "off", "no"} + + +def _debug_write(payload: dict[str, Any]) -> None: + try: + path = debug_log_path() + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as fp: + fp.write(json.dumps(payload, ensure_ascii=False) + "\n") + except Exception: + return + + +# ---- CLS PutLogs 的 protobuf 编码 ---- +# CLS 使用了简化版 protobuf： +# LogGroup { repeated Log logs = 1; optional string filename = 2; } +# Log { optional uint32 time = 1; repeated Content contents = 2; } +# Content { optional string key = 1; optional string value = 2; } + +def _encode_varint(value: int) -> bytes: + result = b"" + while value > 0x7F: + result += bytes([(value & 0x7F) | 0x80]) + value >>= 7 + result += bytes([value & 0x7F]) + return result + +def _encode_field(field_number: int, wire_type: int, data: bytes) -> bytes: + tag = (field_number << 3) | wire_type + return _encode_varint(tag) + data + +def _encode_string_field(field_number: int, value: str) -> bytes: + encoded = value.encode("utf-8") + return _encode_field(field_number, 2, _encode_varint(len(encoded)) + encoded) + +def _encode_uint32_field(field_number: int, value: int) -> bytes: + return _encode_field(field_number, 0, _encode_varint(value)) + +def _encode_content(key: str, value: str) -> bytes: + msg = b"" + if key: + msg += _encode_string_field(1, key) + if value: + msg += _encode_string_field(2, value) + return msg + +def _encode_log(record: dict[str, Any], service_name: str) -> bytes: + ts = record.get("ts") + if isinstance(ts, (int, float)): + log_time = int(ts) if ts < 1_000_000_000_000 else int(ts / 1000) + else: + log_time = int(time.time()) + + msg = _encode_uint32_field(1, log_time) + + merged = {"service_name": service_name, **record} + for key, value in merged.items(): + if value is None: + continue + if isinstance(value, (dict, list)): + value = json.dumps(value, ensure_ascii=False) + else: + value = str(value) + msg += _encode_field(2, 2, _encode_varint(len(_encode_content(key, value))) + _encode_content(key, value)) + + return msg + +def _encode_log_group(records: list[dict[str, Any]], service_name: str) -> bytes: + msg = b"" + for record in records: + log_bytes = _encode_log(record, service_name) + msg += _encode_field(1, 2, _encode_varint(len(log_bytes)) + log_bytes) + if service_name: + msg += _encode_string_field(2, service_name) + return msg + +def _encode_log_group_list(records: list[dict[str, Any]], service_name: str) -> bytes: + """编码 LogGroupList protobuf：`message LogGroupList { repeated LogGroup logGroupList = 1; }`。""" + log_group_bytes = _encode_log_group(records, service_name) + return _encode_field(1, 2, _encode_varint(len(log_group_bytes)) + log_group_bytes) + + +# ---- CLS API v3 使用的 TC3-HMAC-SHA256 签名 ---- + +def _hmac_sha256(key: bytes, data: str) -> bytes: + return hmac.new(key, data.encode("utf-8"), hashlib.sha256).digest() + +def _sha256_hex(data: str) -> str: + return hashlib.sha256(data.encode("utf-8")).hexdigest() + +def _put_logs_via_api( + endpoint: str, + topic_id: str, + secret_id: str, + secret_key: str, + log_group_bytes: bytes, + region: str = "ap-guangzhou", + timeout: int = 20, +) -> tuple[bool, str]: + """通过 CLS API v3 发送 UploadLog，并使用 TC3-HMAC-SHA256 签名。 + + `log_group_bytes` 应该是一个 LogGroupList protobuf。 + """ + host = endpoint + service = "cls" + action = "UploadLog" + version = "2020-10-16" + algorithm = "TC3-HMAC-SHA256" + content_type = "application/octet-stream" + + timestamp = int(time.time()) + date_str = time.strftime("%Y-%m-%d", time.gmtime(timestamp)) + + # 请求体就是原始 protobuf 字节串（LogGroupList） + payload = log_group_bytes + hashed_payload = hashlib.sha256(payload).hexdigest() + + # 规范化请求串 + canonical_headers = f"content-type:{content_type}\nhost:{host}\nx-tc-action:{action.lower()}\n" + signed_headers = "content-type;host;x-tc-action" + canonical_request = f"POST\n/\n\n{canonical_headers}\n{signed_headers}\n{hashed_payload}" + + # 待签名字符串 + credential_scope = f"{date_str}/{service}/tc3_request" + hashed_canonical_request = hashlib.sha256(canonical_request.encode("utf-8")).hexdigest() + string_to_sign = f"{algorithm}\n{timestamp}\n{credential_scope}\n{hashed_canonical_request}" + + # 签名结果 + secret_date = _hmac_sha256(("TC3" + secret_key).encode("utf-8"), date_str) + secret_service = _hmac_sha256(secret_date, service) + secret_signing = _hmac_sha256(secret_service, "tc3_request") + signature = hmac.new(secret_signing, string_to_sign.encode("utf-8"), hashlib.sha256).hexdigest() + + authorization = f"{algorithm} Credential={secret_id}/{credential_scope}, SignedHeaders={signed_headers}, Signature={signature}" + + # 构造 HTTP 请求 + url = f"https://{host}" + headers = { + "Authorization": authorization, + "Content-Type": content_type, + "Host": host, + "X-TC-Action": action, + "X-TC-Timestamp": str(timestamp), + "X-TC-Version": version, + "X-TC-Region": region, + "X-CLS-TopicId": topic_id, + "Content-Length": str(len(payload)), + } + + req = urllib.request.Request(url, data=payload, headers=headers, method="POST") + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + body = resp.read().decode("utf-8", errors="replace") + if resp.status == 200: + return True, body + else: + return False, f"HTTP {resp.status}: {body}" + except urllib.error.HTTPError as e: + body = e.read().decode("utf-8", errors="replace")[:500] + return False, f"HTTP {e.code}: {body}" + except Exception as e: + return False, f"{type(e).__name__}: {e}" + + +def _int_value( + name: str, + *, + env_local: dict[str, str], + env_file: dict[str, str], + default: int, + aliases: tuple[str, ...] = (), +) -> int: + raw = _value(name, env_local=env_local, env_file=env_file, default=str(default), aliases=aliases) + try: + parsed = int(raw) + except Exception: + return default + return parsed if parsed > 0 else default + + +def load_config(cwd: str | None = None) -> CLSConfig: + project_root = bootstrap.project_root(cwd) + env_file = bootstrap.read_env_file(project_root / ".env") + env_local = bootstrap.read_env_file(project_root / ".env.local") + return CLSConfig( + enabled=_enabled_flag(env_local=env_local, env_file=env_file), + endpoint=_value( + "CLS_ENDPOINT", + env_local=env_local, + env_file=env_file, + ), + topic_id=_value( + "CLS_TOPIC_ID", + env_local=env_local, + env_file=env_file, + ), + secret_id=_value( + "CLS_SECRET_ID", + env_local=env_local, + env_file=env_file, + aliases=( + "TC_SECRET_ID", + "TENCENTCLOUD_SECRET_ID", + "TENCENTCLOUD_SECRET_ID_438167613", + ), + ), + secret_key=_value( + "CLS_SECRET_KEY", + env_local=env_local, + env_file=env_file, + aliases=( + "TC_SECRET_KEY", + "TENCENTCLOUD_SECRET_KEY", + "TENCENTCLOUD_SECRET_KEY_438167613", + ), + ), + secret_token=_value( + "CLS_SECRET_TOKEN", + env_local=env_local, + env_file=env_file, + aliases=("TC_SECRET_TOKEN", "TC_SESSION_TOKEN", "CLS_SESSION_TOKEN"), + ), + service_name=_value( + "CLS_SERVICE_NAME", + env_local=env_local, + env_file=env_file, + ), + timeout_seconds=_int_value( + "CLS_UPLOAD_TIMEOUT_SECONDS", + env_local=env_local, + env_file=env_file, + default=0, + aliases=("CLS_TIMEOUT_SECONDS",), + ), + helper_path=helper_path(), + sdk_entry_path=sdk_entry_path(), + ) + + +def mirror_record(record: dict[str, Any], *, cwd: str | None = None) -> bool: + config = load_config(cwd) + if not config.ready: + _debug_write( + { + "stage": "config_not_ready", + "endpoint": config.endpoint, + "topic_id": config.topic_id, + "secret_id_present": bool(config.secret_id), + "secret_key_present": bool(config.secret_key), + "secret_token_present": bool(config.secret_token), + "timeout_seconds": config.timeout_seconds, + "helper_exists": config.helper_path.is_file(), + "sdk_exists": config.sdk_entry_path.is_file(), + "event": record.get("event"), + "sid": record.get("sid"), + } + ) + return False + + _debug_write( + { + "stage": "uploader_start", + "endpoint": config.endpoint, + "topic_id": config.topic_id, + "timeout_seconds": config.timeout_seconds, + "event": record.get("event"), + "sid": record.get("sid"), + } + ) + + # 先尝试 Python 原生 API v3 上报（兼容内网 endpoint） + try: + log_group_bytes = _encode_log_group_list([record], config.service_name) + success, detail = _put_logs_via_api( + endpoint=config.endpoint, + topic_id=config.topic_id, + secret_id=config.secret_id, + secret_key=config.secret_key, + log_group_bytes=log_group_bytes, + timeout=config.timeout_seconds, + ) + if success: + _debug_write( + { + "stage": "uploader_ok", + "endpoint": config.endpoint, + "topic_id": config.topic_id, + "method": "python_api_v3", + "event": record.get("event"), + "sid": record.get("sid"), + } + ) + return True + else: + _debug_write( + { + "stage": "uploader_failed", + "method": "python_api_v3", + "error": detail[:2000], + "timeout_seconds": config.timeout_seconds, + "event": record.get("event"), + "sid": record.get("sid"), + } + ) + except Exception as err: + _debug_write( + { + "stage": "uploader_exception", + "method": "python_api_v3", + "error": f"{type(err).__name__}: {err}", + "timeout_seconds": config.timeout_seconds, + "event": record.get("event"), + "sid": record.get("sid"), + } + ) + + # 回退方案：使用 node SDK uploader + payload = { + "endpoint": config.endpoint, + "topicId": config.topic_id, + "secretId": config.secret_id, + "secretKey": config.secret_key, + "secretToken": config.secret_token, + "serviceName": config.service_name, + "records": [record], + } + try: + completed = subprocess.run( + ["node", str(config.helper_path), str(config.sdk_entry_path)], + input=json.dumps(payload, ensure_ascii=False), + text=True, + capture_output=True, + check=False, + timeout=config.timeout_seconds, + ) + if completed.returncode != 0: + _debug_write( + { + "stage": "uploader_failed", + "returncode": completed.returncode, + "stdout": (completed.stdout or "")[:2000], + "stderr": (completed.stderr or "")[:2000], + "timeout_seconds": config.timeout_seconds, + "event": record.get("event"), + "sid": record.get("sid"), + } + ) + return False + except Exception as err: + _debug_write( + { + "stage": "uploader_exception", + "error": f"{type(err).__name__}: {err}", + "timeout_seconds": config.timeout_seconds, + "event": record.get("event"), + "sid": record.get("sid"), + } + ) + return False + + _debug_write( + { + "stage": "uploader_ok", + "endpoint": config.endpoint, + "topic_id": config.topic_id, + "timeout_seconds": config.timeout_seconds, + "event": record.get("event"), + "sid": record.get("sid"), + } + ) + return True diff --git a/.cursor/skills/agent-observability/scripts/core/cls_uploader.mjs b/.cursor/skills/agent-observability/scripts/core/cls_uploader.mjs new file mode 100644 index 0000000..5c5dccc --- /dev/null +++ b/.cursor/skills/agent-observability/scripts/core/cls_uploader.mjs @@ -0,0 +1,96 @@ +import { createRequire } from 'node:module'; + +const MAX_FIELD_LENGTH = 32 * 1024; + +function truncate(value) { + if (value.length <= MAX_FIELD_LENGTH) { + return value; + } + return `${value.slice(0, MAX_FIELD_LENGTH)}...[truncated]`; +} + +function stringifyValue(value) { + if (value === null || value === undefined) { + return ''; + } + if (typeof value === 'string') { + return truncate(value); + } + if (typeof value === 'number' || typeof value === 'boolean') { + return String(value); + } + try { + return truncate(JSON.stringify(value)); + } catch { + return truncate(String(value)); + } +} + +function timestampSeconds(record) { + const raw = Number(record?.ts); + if (!Number.isFinite(raw) || raw <= 0) { + return Math.floor(Date.now() / 1000); + } + return raw > 1_000_000_000_000 ? Math.floor(raw / 1000) : Math.floor(raw); +} + +async function readStdin() { + const chunks = []; + for await (const chunk of process.stdin) { + chunks.push(chunk); + } + return Buffer.concat(chunks).toString('utf8'); +} + +async function main() { + const sdkPath = process.argv[2]; + if (!sdkPath) { + process.exitCode = 2; + process.stderr.write('missing sdk path\n'); + return; + } + + const require = createRequire(import.meta.url); + const { AsyncClient, LogItem, Content, LogGroup, PutLogsRequest } = require(sdkPath); + + const raw = await readStdin(); + const payload = JSON.parse(raw || '{}'); + const records = Array.isArray(payload.records) ? payload.records : []; + if (!payload.endpoint || !payload.topicId || !payload.secretId || !payload.secretKey || records.length === 0) { + process.exitCode = 0; + return; + } + + const client = new AsyncClient({ + endpoint: payload.endpoint, + secretId: payload.secretId, + secretKey: payload.secretKey, + secretToken: payload.secretToken || '', + sourceIp: '127.0.0.1', + retry_times: 3, + }); + + const logGroup = new LogGroup(); + logGroup.setFilename(payload.serviceName || 'agent-observability'); + + for (const record of records) { + const item = new LogItem(); + const merged = { + service_name: payload.serviceName || 'agent-observability', + ...record, + }; + for (const [key, value] of Object.entries(merged)) { + item.pushBack(new Content(key, stringifyValue(value))); + } + item.setTime(timestampSeconds(record)); + logGroup.addLogs(item); + } + + const request = new PutLogsRequest(payload.topicId, logGroup); + await client.PutLogs(request); +} + +main().catch((error) => { + process.exitCode = 1; + process.stderr.write(`${error?.message || String(error)}\n`); +}); diff --git a/.cursor/skills/agent-observability/scripts/core/collector.py b/.cursor/skills/agent-observability/scripts/core/collector.py new file mode 100644 index 0000000..cfd681b --- /dev/null +++ b/.cursor/skills/agent-observability/scripts/core/collector.py @@ -0,0 +1,1008 @@ +"""通用采集层。 + +这一层负责最基础的观测数据采集： +- 维护 Pre/PostToolUse 的配对关系，计算工具耗时 +- 解析 transcript 增量，提取 usage / model / tool 记录 +- 把 skill/rule 命中情况写入 session state +""" +from __future__ import annotations + +import json +import os +import time +from pathlib import Path +from typing import Any + +from . import agent_identity, scanner, state as st + +# 需要从 transcript 中提取的 token 字段 +TOKEN_KEYS = ( + "input_tokens", "output_tokens", "cache_read_input_tokens", + "cache_creation_input_tokens", "total_tokens", +) + +# transcript 记录里可能携带模型名的字段 +MODEL_KEYS = ("model", "requestModelName", "requestModelId") +MESSAGE_ID_KEYS = ("messageId", "responseId", "requestId") + + +def transcript_path(data: dict[str, Any]) -> str: + """从 hook payload 中提取 transcript 路径字段。""" + return str(data.get("transcript_path") or "") + + +def read_stdin_json() -> dict: + """从 stdin 读取 JSON；失败时返回空字典。""" + import sys + try: + raw = sys.stdin.read() + if raw.strip(): + d = json.loads(raw) + if isinstance(d, dict): + return d + return {} + except Exception as e: + return {"_parse_error": str(e)} + + +def record_pre(pending_path: Path, data: dict) -> None: + """记录一条 PreToolUse，供后续 PostToolUse 计算耗时。""" + pending = st.load_pending(pending_path) + sid = data.get("session_id", "?") + tool = data.get("tool_name", "?") + key = f"{sid}::{tool}::{time.time_ns()}" + pending[key] = { + "start": time.time(), + "tool_name": tool, + "session_id": sid, + } + # 只保留最近 20 条 pending 记录 + if len(pending) > 20: + for k in list(pending.keys())[:-20]: + pending.pop(k, None) + st.save_pending(pending_path, pending) + + +def record_post(pending_path: Path, data: dict) -> int | None: + """把 PostToolUse 与之前的 PreToolUse 配对，并返回耗时毫秒数。""" + pending = st.load_pending(pending_path) + tool = data.get("tool_name") + sid = data.get("session_id", "?") + candidates = [(k, v) for k, v in pending.items() + if v.get("session_id") == sid and v.get("tool_name") == tool] + if not candidates: + return None + candidates.sort(key=lambda kv: kv[1]["start"]) + key, item = candidates[-1] + duration_ms = int((time.time() - item["start"]) * 1000) + pending.pop(key, None) + st.save_pending(pending_path, pending) + return duration_ms + + +def parse_transcript_tail( + path: str, + max_lines: int = 50, + last_offset: int = 0, +) -> dict[str, Any]: + """解析 transcript 的增量 JSONL 内容，提取 usage、model 与 tool 元数据。 + + 返回结构： + { + "tokens": {...} or None, # 最新累计 usage + "prev_tokens": {...} or None, # 倒数第二条累计 usage（兼容字段） + "model": str or None, + "offset": int, # 当前 EOF + "tool_records": [ + {"offset": int, "timestamp_ms": int | None, "tool": str, "call_id": str | None, "message_id": str | None, "kind": str}, + ... + ], + "usage_records": [ + {"offset": int, "tokens": {...}, "model": str | None, "message_id": str | None}, + ... + ], + } + + ``usage_records`` 会保留 ``last_offset`` 之后发现的全部 usage 项， + 这样调用方可以回放窗口内每一次模型请求，而不只是最后一个快照。 + """ + summary: dict[str, Any] = { + "tokens": None, + "prev_tokens": None, + "model": None, + "offset": 0, + "tool_records": [], + "usage_records": [], + } + if not path or not os.path.isfile(path): + return summary + + try: + file_size = os.path.getsize(path) + summary["offset"] = file_size + if file_size <= last_offset: + return summary + + with open(path, "rb") as fp: + read_start = max(0, int(last_offset or 0)) + fp.seek(read_start) + tail_bytes = fp.read() + except Exception: + return summary + + last_model_seen: str | None = None + usage_history: list[dict[str, Any]] = [] + pending_tool_records: list[dict[str, Any]] = [] + cursor = max(0, int(last_offset or 0)) + + for raw_line in tail_bytes.splitlines(keepends=True): + cursor += len(raw_line) + if not raw_line.lstrip().startswith(b"{"): + continue + try: + obj = json.loads(raw_line.decode("utf-8", errors="ignore")) + except Exception: + continue + + rec_type = str(obj.get("type") or "").strip().lower() if isinstance(obj, dict) else "" + message_id = _find_message_id(obj) + m = _find_model(obj) + if m: + last_model_seen = m + line_tool_records = _find_tool_records(obj) + if message_id and rec_type not in {"function_call", "function_call_result"}: + for pending_record in pending_tool_records: + pending_record["next_message_id"] = message_id + pending_tool_records = [] + for tool_record in line_tool_records: + tool_record["offset"] = cursor + tool_record["timestamp_ms"] = _find_timestamp_ms(obj) + tool_record["message_id"] = message_id + summary["tool_records"].append(tool_record) + pending_tool_records.append(tool_record) + usage = _find_usage(obj) + if not usage: + continue + + usage_history.append({ + "offset": cursor, + "tokens": usage, + "model": m or last_model_seen, + "message_id": message_id, + }) + if m: + summary["model"] = m + + if usage_history: + summary["usage_records"] = usage_history + summary["tokens"] = usage_history[-1]["tokens"] + if len(usage_history) >= 2: + summary["prev_tokens"] = usage_history[-2]["tokens"] + + if not summary.get("model") and last_model_seen: + summary["model"] = last_model_seen + return summary + + +def record_tool_usage( + state_path: Path, + sid: str, + data: dict, + skills_meta: dict, + rules_meta: dict, + active_agent: str | None = None, + collect_skills: bool = False, +) -> tuple[list[str], list[str]]: + """把一次 tool 调用命中的 skill/rule 写入 session state。 + + Returns: + (used_skills, used_rules): 本次工具调用命中的 skill/rule 名称列表。 + """ + tool = data.get("tool_name") + + def _update(state: dict[str, Any]) -> tuple[list[str], list[str]]: + sess = st.ensure_session(state, sid) + agent = active_agent or sess.get("current_agent") or "main" + + used_skills: set[str] = set() + used_rules: set[str] = set() + + # 1) 可选的 skill 收集 + if collect_skills: + # 直接 use_skill 调用 + direct = scanner.extract_skill_from_tool_call(data) + if direct: + st.bump_skill(sess, direct, via="use_skill", tool=tool, + meta=skills_meta.get(direct), agent=agent) + used_skills.add(direct) + + # 路径推断 + path_skills, _ = scanner.extract_paths_from_tool_call(data) + for s in path_skills: + st.bump_skill(sess, s, via="path-inferred", tool=tool, + meta=skills_meta.get(s), agent=agent) + used_skills.add(s) + + # 子模块命中 + for hit in scanner.extract_submodule_hits(data): + skill_name = hit["skill"] + # unknown 仅代表“命中子模块但无法确定 skill 名称”，不写入实时 skill 字段 + if not skill_name or skill_name == "unknown": + continue + st.bump_skill(sess, skill_name, via="submodule", tool=tool, + meta=skills_meta.get(skill_name), + submodule=hit["submodule"], agent=agent) + used_skills.add(skill_name) + + # Bash skill 脚本 + for s in scanner.extract_bash_skill_scripts(data): + st.bump_skill(sess, s, via="bash-script", tool=tool, + meta=skills_meta.get(s), agent=agent) + used_skills.add(s) + + # 2) 基于路径推断的 rule（始终开启） + _, path_rules = scanner.extract_paths_from_tool_call(data) + for r in path_rules: + st.bump_rule(sess, r, via="path-inferred", tool=tool, meta=rules_meta.get(r), agent=agent) + used_rules.add(r) + + # 3) 当前激活的 rule（始终开启） + for r in scanner.active_rules_for_call(data, rules_meta, active_agent=agent): + st.bump_rule(sess, r, via="active-rule", tool=tool, meta=rules_meta.get(r), agent=agent) + used_rules.add(r) + + st.prune_sessions(state) + return sorted(used_skills), sorted(used_rules) + + return st.update_state_locked(state_path, _update) + + + +def cache_inventory( + state_path: Path, + sid: str, + skills_meta: dict, + rules_meta: dict, +) -> None: + """把完整的 skill/rule inventory 缓存在 session state 中（在 SessionStart 调用）。""" + def _update(state: dict[str, Any]) -> None: + sess = st.ensure_session(state, sid) + sess["_skills_meta"] = skills_meta + sess["_rules_meta"] = rules_meta + sess["_inventory_scanned_at"] = time.time() + + # 为所有已知 skill/rule 预先初始化 count=0 的 usage 记录 + for name, meta in skills_meta.items(): + rec = sess["skills"].setdefault(name, { + "count": 0, "first_ts": None, "last_ts": None, + "tools": [], "via": ["static-scanned"], + "source": meta.get("source"), "version": meta.get("version"), + }) + if not rec.get("source"): + rec["source"] = meta.get("source") + if "static-scanned" not in rec.get("via", []): + rec.setdefault("via", []).append("static-scanned") + + for name, meta in rules_meta.items(): + rec = sess["rules"].setdefault(name, { + "count": 0, "first_ts": None, "last_ts": None, + "tools": [], "via": ["static-scanned"], "source": meta.get("source"), + }) + if "static-scanned" not in rec.get("via", []): + rec.setdefault("via", []).append("static-scanned") + if not rec.get("source"): + rec["source"] = meta.get("source") + + st.prune_sessions(state) + + st.update_state_locked(state_path, _update) + + +def load_cached_inventory(state_path: Path, sid: str) -> tuple[dict, dict]: + """从 state 中读取缓存的 skill/rule inventory，避免在热路径里执行 rglob。""" + state = st.load_state(state_path) + sess = state.get(sid, {}) + if not isinstance(sess, dict): + return {}, {} + return ( + sess.get("_skills_meta") or {}, + sess.get("_rules_meta") or {}, + ) + + +def get_session_usage(state_path: Path, sid: str) -> tuple[dict, dict]: + """返回某个 session 的 skill/rule usage 字典。""" + state = st.load_state(state_path) + sess = state.get(sid, {}) + if not isinstance(sess, dict): + return {}, {} + return sess.get("skills", {}) or {}, sess.get("rules", {}) or {} + +def related_transcript_paths(sid: str, transcript_path: str) -> list[str]: + """收集当前 session 的主 transcript 以及所有 subagent transcript。""" + paths: list[Path] = [] + current = Path(transcript_path).expanduser() if transcript_path else None + if current and current.is_file(): + paths.append(current) + + bundle_dir: Path | None = None + if current: + if current.name == f"{sid}.jsonl": + candidate = current.with_suffix("") + if candidate.is_dir(): + bundle_dir = candidate + else: + for parent in [current.parent, *current.parents]: + if parent.name == sid and parent.is_dir(): + bundle_dir = parent + break + + if bundle_dir is None and current: + candidate = current.parent / sid + if candidate.is_dir(): + bundle_dir = candidate + + if bundle_dir is not None: + main_transcript = bundle_dir.with_suffix(".jsonl") + if main_transcript.is_file(): + paths.append(main_transcript) + subagents_dir = bundle_dir / "subagents" + if subagents_dir.is_dir(): + paths.extend(sorted(path for path in subagents_dir.glob("*.jsonl") if path.is_file())) + + out: list[str] = [] + seen: set[str] = set() + for path in paths: + resolved = str(path.resolve()) + if resolved in seen: + continue + seen.add(resolved) + out.append(resolved) + return out + + +def _transcript_size_signature(paths: list[str]) -> tuple[tuple[str, int], ...]: + """基于路径和文件大小生成 transcript 稳定性签名。""" + signature: list[tuple[str, int]] = [] + for path in paths: + try: + size = Path(path).stat().st_size + except Exception: + size = -1 + signature.append((path, int(size))) + return tuple(signature) + + +def settled_related_transcript_paths( + sid: str, + transcript_path: str, + *, + max_wait_s: float = 2.0, + interval_s: float = 0.4, + stable_rounds: int = 2, +) -> list[str]: + """在 replay 前短暂等待 transcript 文件停止增长。""" + deadline = time.monotonic() + max(0.0, max_wait_s) + previous: tuple[tuple[str, int], ...] | None = None + stable_count = 0 + paths = related_transcript_paths(sid, transcript_path) + + while True: + paths = related_transcript_paths(sid, transcript_path) + current = _transcript_size_signature(paths) + if current == previous: + stable_count += 1 + else: + previous = current + stable_count = 0 + if stable_count >= stable_rounds or time.monotonic() >= deadline: + return paths + sleep_for = min(interval_s, max(0.0, deadline - time.monotonic())) + if sleep_for <= 0: + return paths + time.sleep(sleep_for) + + +def session_id_for_transcript_path(transcript_path: str) -> str | None: + """从 transcript 文件内容中反查 session id。""" + path = Path(transcript_path) + try: + with path.open("r", encoding="utf-8") as fp: + for _ in range(8): + line = fp.readline() + if not line: + break + try: + record = json.loads(line) + except Exception: + continue + session_id = record.get("sessionId") + if isinstance(session_id, str) and session_id.strip(): + return session_id.strip() + except Exception: + return None + return None + + +def resolve_transcript_path_alias(sid: str, transcript_path: str) -> str: + """把别名 transcript 路径解析成当前 session 下的真实文件路径。""" + if not transcript_path: + return "" + current = Path(transcript_path).expanduser() + if current.is_file(): + return str(current.resolve()) + + alias_session_id = current.stem if current.suffix == ".jsonl" else "" + if not alias_session_id or alias_session_id == sid: + return str(current) + + for candidate in related_transcript_paths(sid, transcript_path): + if session_id_for_transcript_path(candidate) == alias_session_id: + return candidate + return str(current) + +def compute_usage_delta(current: dict[str, int], prev: dict[str, int] | None) -> dict[str, int]: + """把一条 usage 记录归一化成当前要发出的 event payload。""" + _ = prev + out = dict(current) + out["total"] = int(out.get("input", 0) or 0) + int(out.get("output", 0) or 0) + return out + + +def normalize_tokens(tokens: dict[str, Any] | None) -> dict[str, int] | None: + """兼容多种 token 字段命名，并统一折叠成一套 schema。""" + if not isinstance(tokens, dict): + return None + + def _int(value: Any) -> int: + try: + return int(value or 0) + except Exception: + return 0 + + input_t = _int(tokens.get("input") if "input" in tokens else tokens.get("input_tokens")) + output_t = _int(tokens.get("output") if "output" in tokens else tokens.get("output_tokens")) + cache_read = _int(tokens.get("cache_read") if "cache_read" in tokens else tokens.get("cache_read_input_tokens")) + cache_creation = _int(tokens.get("cache_creation") if "cache_creation" in tokens else tokens.get("cache_creation_input_tokens")) + total = input_t + output_t + if input_t <= 0 and output_t <= 0 and total <= 0: + return None + out = {"input": input_t, "output": output_t, "cache_read": cache_read, "total": total} + if cache_creation > 0: + out["cache_creation"] = cache_creation + return out + + +def collect_transcript_entries( + state_path: Path, + sid: str, + transcript_path: str, + *, + max_lines: int, +) -> dict[str, Any]: + """读取 transcript 增量，并返回带 source offset 的解析结果。""" + state_data = st.load_state(state_path) + last_offset = st.get_transcript_offset(state_data, sid, transcript_path) + tail = parse_transcript_tail(transcript_path, max_lines=max_lines, last_offset=last_offset) + new_offset = int(tail.get("offset", 0) or 0) + + entries: list[dict[str, Any]] = [] + tool_records: list[dict[str, Any]] = [] + last_cumulative = st.get_last_cumulative_usage(state_data, sid, transcript_path) or None + usage_records = tail.get("usage_records") if isinstance(tail, dict) else None + if not isinstance(usage_records, list): + usage_records = [] + raw_tool_records = tail.get("tool_records") if isinstance(tail, dict) else None + if not isinstance(raw_tool_records, list): + raw_tool_records = [] + + for raw_entry in usage_records: + if not isinstance(raw_entry, dict): + continue + current = normalize_tokens(raw_entry.get("tokens")) + if not current: + continue + delta = compute_usage_delta(current, last_cumulative) + entries.append({ + "offset": int(raw_entry.get("offset", 0) or 0), + "tokens": delta, + "model": str(raw_entry.get("model") or tail.get("model") or "") or None, + "message_id": str(raw_entry.get("message_id") or "").strip() or None, + }) + last_cumulative = current + + for raw_tool_record in raw_tool_records: + if not isinstance(raw_tool_record, dict): + continue + tool_name = str(raw_tool_record.get("tool") or "").strip() + if not tool_name: + continue + tool_records.append({ + "offset": int(raw_tool_record.get("offset", 0) or 0), + "timestamp_ms": int(raw_tool_record.get("timestamp_ms", 0) or 0) or None, + "tool": tool_name, + "call_id": str(raw_tool_record.get("call_id") or "").strip() or None, + "kind": str(raw_tool_record.get("kind") or "").strip() or None, + "message_id": str(raw_tool_record.get("message_id") or "").strip() or None, + "next_message_id": str(raw_tool_record.get("next_message_id") or "").strip() or None, + "arguments": raw_tool_record.get("arguments"), + "arguments_display_text": raw_tool_record.get("arguments_display_text"), + "result_content": raw_tool_record.get("result_content"), + "raw_response": raw_tool_record.get("raw_response"), + "output_text": raw_tool_record.get("output_text"), + }) + + return { + "entries": entries, + "tool_records": tool_records, + "new_offset": new_offset, + "last_cumulative": last_cumulative, + } + + +def find_current_tool_message_id( + state_path: Path, + sid: str, + transcript_path: str, + tool_name: str, + *, + max_lines: int = 80, +) -> str | None: + """找到当前 tool 事件应该归属的 message id。""" + if not transcript_path or not tool_name: + return None + scan = collect_transcript_entries( + state_path=state_path, + sid=sid, + transcript_path=transcript_path, + max_lines=max_lines, + ) + tool_records = scan.get("tool_records") or [] + latest_message_id: str | None = None + latest_offset = -1 + for rec in tool_records: + if not isinstance(rec, dict): + continue + if str(rec.get("tool") or "") != str(tool_name): + continue + message_id = ( + str(rec.get("next_message_id") or "").strip() + or str(rec.get("message_id") or "").strip() + or None + ) + if not message_id: + continue + offset = int(rec.get("offset", 0) or 0) + if offset >= latest_offset: + latest_offset = offset + latest_message_id = message_id + return latest_message_id + + +def tool_details_from_record(rec: dict[str, Any]) -> dict[str, Any]: + """把原始 tool 记录投影成日志里使用的紧凑 detail 结构。""" + details: dict[str, Any] = {} + call_id = str(rec.get("call_id") or "").strip() or None + if call_id: + details["call_id"] = call_id + arguments = rec.get("arguments") + if arguments is not None: + details["arguments"] = arguments + arguments_display_text = rec.get("arguments_display_text") + if arguments_display_text is not None: + details["arguments_display_text"] = arguments_display_text + result_content = rec.get("result_content") + if result_content is not None: + details["result_content"] = result_content + raw_response = rec.get("raw_response") + if isinstance(raw_response, dict) and raw_response: + details["raw_response"] = raw_response + output_text = rec.get("output_text") + if output_text is not None: + details["output_text"] = output_text + next_message_id = str(rec.get("next_message_id") or "").strip() or None + original_message_id = str(rec.get("message_id") or "").strip() or None + if next_message_id and original_message_id: + details["original_message_id"] = original_message_id + if next_message_id: + details["next_message_id"] = next_message_id + if original_message_id and next_message_id and original_message_id != next_message_id: + details["message_id_reassigned"] = True + return details + + +def tool_record_merge_key(rec: dict[str, Any]) -> str: + """为一条 tool record 生成去重合并时使用的稳定 key。""" + call_id = str(rec.get("call_id") or "").strip() + if call_id: + return f"call_id:{call_id}" + offset = int(rec.get("offset", 0) or 0) + tool = str(rec.get("tool") or "").strip() + message_id = str(rec.get("message_id") or "").strip() + timestamp_ms = int(rec.get("timestamp_ms", 0) or 0) + return f"fallback:{tool}:{message_id}:{timestamp_ms}:{offset}" + + +def merge_tool_records(tool_records: list[dict[str, Any]]) -> list[dict[str, Any]]: + """对指向同一次调用的 tool 记录做去重合并。""" + merged: dict[str, dict[str, Any]] = {} + order: list[str] = [] + for rec in tool_records: + if not isinstance(rec, dict): + continue + key = tool_record_merge_key(rec) + if key not in merged: + merged[key] = { + "offset": int(rec.get("offset", 0) or 0), + "timestamp_ms": int(rec.get("timestamp_ms", 0) or 0) or None, + "tool": str(rec.get("tool") or "").strip(), + "call_id": str(rec.get("call_id") or "").strip() or None, + "kind": str(rec.get("kind") or "").strip() or None, + "message_id": str(rec.get("message_id") or "").strip() or None, + "next_message_id": str(rec.get("next_message_id") or "").strip() or None, + "arguments": rec.get("arguments"), + "arguments_display_text": rec.get("arguments_display_text"), + "result_content": rec.get("result_content"), + "raw_response": rec.get("raw_response"), + "output_text": rec.get("output_text"), + } + order.append(key) + continue + + current = merged[key] + current["offset"] = max(int(current.get("offset", 0) or 0), int(rec.get("offset", 0) or 0)) + current_ts = int(current.get("timestamp_ms", 0) or 0) + rec_ts = int(rec.get("timestamp_ms", 0) or 0) + if current_ts <= 0 and rec_ts > 0: + current["timestamp_ms"] = rec_ts + for field in ("tool", "call_id", "kind", "message_id", "next_message_id", "arguments", "arguments_display_text", "result_content", "raw_response", "output_text"): + if current.get(field) is None and rec.get(field) is not None: + current[field] = rec.get(field) + if rec.get("arguments") is not None: + current["arguments"] = rec.get("arguments") + if rec.get("arguments_display_text") is not None: + current["arguments_display_text"] = rec.get("arguments_display_text") + if rec.get("result_content") is not None: + current["result_content"] = rec.get("result_content") + if rec.get("raw_response") is not None: + current["raw_response"] = rec.get("raw_response") + if rec.get("output_text") is not None: + current["output_text"] = rec.get("output_text") + + return [merged[key] for key in order] + + +def tool_context_claim_key(rec: dict[str, Any]) -> str: + """为 tool context 生成 claim 去重键，避免重复消费。""" + call_id = str(rec.get("call_id") or "").strip() + if call_id: + return f"tool_call|{call_id}" + tool = str(rec.get("tool") or "").strip() + message_id = str(rec.get("message_id") or "").strip() + timestamp_ms = int(rec.get("timestamp_ms", 0) or 0) + offset = int(rec.get("offset", 0) or 0) + return f"tool_call|{tool}|{message_id}|{timestamp_ms}|{offset}" + + +def find_current_tool_context( + state_path: Path, + sid: str, + transcript_path: str, + tool_name: str, + *, + call_id: str | None = None, + event_ts: float | None = None, + max_lines: int = 80, + claim: bool = False, +) -> dict[str, Any] | None: + """为当前 tool 调用解析最合适的 transcript 上下文块。""" + if not tool_name: + return None + event_ts_ms = int(float(event_ts) * 1000) if event_ts else None + normalized_call_id = str(call_id or "").strip() or None + + resolved_path = resolve_transcript_path_alias(sid, transcript_path) + candidate_paths: list[str] = [] + if resolved_path: + candidate_paths.append(resolved_path) + for path in related_transcript_paths(sid, transcript_path): + if path not in candidate_paths: + candidate_paths.append(path) + + candidates: list[tuple[tuple[int, int, int], dict[str, Any]]] = [] + for path in candidate_paths: + scan = collect_transcript_entries( + state_path=state_path, + sid=sid, + transcript_path=path, + max_lines=max_lines, + ) + tool_records = merge_tool_records(scan.get("tool_records") or []) + for rec in tool_records: + if not isinstance(rec, dict): + continue + if str(rec.get("tool") or "") != str(tool_name): + continue + rec_call_id = str(rec.get("call_id") or "").strip() or None + if normalized_call_id and rec_call_id != normalized_call_id: + continue + message_id = ( + str(rec.get("next_message_id") or "").strip() + or str(rec.get("message_id") or "").strip() + or None + ) + if not message_id: + continue + offset = int(rec.get("offset", 0) or 0) + timestamp_ms = int(rec.get("timestamp_ms", 0) or 0) + if normalized_call_id: + score = (0, 0, -offset) + elif event_ts_ms and timestamp_ms > 0: + delta = timestamp_ms - event_ts_ms + score = (abs(delta), 0 if delta >= 0 else 1, -offset) + else: + score = (10**12, 1, -offset) + candidates.append(( + score, + { + "message_id": message_id, + "transcript_path": path, + "agent": agent_identity.agent_for_transcript_path(path, "main"), + "pt_id": agent_identity.pt_id_from_transcript_path(path), + "tool_details": tool_details_from_record(rec), + "_claim_key": tool_context_claim_key(rec), + }, + )) + + candidates.sort(key=lambda item: item[0]) + saw_duplicate = False + for _, candidate in candidates: + if not claim: + out = { + "message_id": str(candidate.get("message_id") or ""), + "transcript_path": str(candidate.get("transcript_path") or ""), + "agent": str(candidate.get("agent") or "main"), + "tool_details": dict(candidate.get("tool_details") or {}), + } + pt_id = candidate.get("pt_id") + if pt_id: + out["pt_id"] = pt_id + return out + if st.claim_transcript_event( + state_path, + sid, + str(candidate.get("_claim_key") or ""), + str(candidate.get("transcript_path") or "") or None, + ): + out = { + "message_id": str(candidate.get("message_id") or ""), + "transcript_path": str(candidate.get("transcript_path") or ""), + "agent": str(candidate.get("agent") or "main"), + "tool_details": dict(candidate.get("tool_details") or {}), + } + pt_id = candidate.get("pt_id") + if pt_id: + out["pt_id"] = pt_id + return out + saw_duplicate = True + + if saw_duplicate: + return {"duplicate": True} + return None + + +def extract_tool_call_id(data: dict[str, Any]) -> str | None: + """从事件 payload 或 tool_input 中提取 tool call id。""" + for key in ("call_id", "callId", "tool_call_id", "toolCallId"): + value = data.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + + tool_input = data.get("tool_input") + if isinstance(tool_input, dict): + for key in ("call_id", "callId", "tool_call_id", "toolCallId", "id"): + value = tool_input.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + return None + + +def find_fallback_usage_event( + tool_event: dict[str, Any], + usage_events: list[dict[str, Any]], +) -> dict[str, Any] | None: + """当找不到直接匹配的 tool usage 时，为 AgentLens 选一条兜底 usage。""" + tool_name = str(tool_event.get("tool") or "").strip() + transcript_path = str(tool_event.get("transcript_path") or "").strip() + agent = str(tool_event.get("agent") or "main").strip() or "main" + + strong_matches: list[dict[str, Any]] = [] + weak_matches: list[dict[str, Any]] = [] + for usage_event in usage_events: + if not isinstance(usage_event, dict): + continue + if (str(usage_event.get("agent") or "main").strip() or "main") != agent: + continue + if transcript_path and str(usage_event.get("transcript_path") or "").strip() != transcript_path: + continue + if not str(usage_event.get("message_id") or "").strip(): + continue + + weak_matches.append(usage_event) + event_tool = str(usage_event.get("tool") or "").strip() + if tool_name and event_tool in {tool_name, "model_request"}: + strong_matches.append(usage_event) + + if strong_matches: + return strong_matches[-1] + if weak_matches: + return weak_matches[-1] + return None + + +def build_transcript_event_key(*, kind: str, tool: str, entry: dict[str, Any]) -> str: + """为 transcript 重放出的事件生成幂等键。""" + _ = (kind, tool) + return "|".join(["model_request", str(entry.get("offset", 0) or 0)]) + +def _find_usage(obj: Any, _depth: int = 0) -> dict | None: + """递归查找 JSON 对象里的 token usage。""" + if _depth > 10: + return None + if isinstance(obj, dict): + if "usage" in obj and isinstance(obj["usage"], dict): + u = obj["usage"] + picked = {k: u[k] for k in TOKEN_KEYS if k in u} + if picked: + return picked + for v in obj.values(): + r = _find_usage(v, _depth + 1) + if r: + return r + elif isinstance(obj, list): + for v in obj: + r = _find_usage(v, _depth + 1) + if r: + return r + return None + + +def _find_model(obj: Any, _depth: int = 0) -> str | None: + """递归查找 JSON 对象里的非空模型名。 + + 优先使用 `providerData.model` 或顶层 `model`， + 其次回退到 `requestModelName` / `requestModelId`。 + """ + if _depth > 10: + return None + if isinstance(obj, dict): + for k in MODEL_KEYS: + v = obj.get(k) + if isinstance(v, str) and v.strip(): + return v.strip() + for v in obj.values(): + r = _find_model(v, _depth + 1) + if r: + return r + elif isinstance(obj, list): + for v in obj: + r = _find_model(v, _depth + 1) + if r: + return r + return None + + +def _find_message_id(obj: Any, _depth: int = 0) -> str | None: + """递归查找稳定的 message/request 分组标识。""" + if _depth > 10: + return None + if isinstance(obj, dict): + provider = obj.get("providerData") + if isinstance(provider, dict): + for key in MESSAGE_ID_KEYS: + value = provider.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + extra = obj.get("extra") + if isinstance(extra, dict): + for key in MESSAGE_ID_KEYS: + value = extra.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + for key in MESSAGE_ID_KEYS: + value = obj.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + for v in obj.values(): + r = _find_message_id(v, _depth + 1) + if r: + return r + elif isinstance(obj, list): + for v in obj: + r = _find_message_id(v, _depth + 1) + if r: + return r + return None + + +def _find_timestamp_ms(obj: Any, _depth: int = 0) -> int | None: + """递归查找 transcript 中的毫秒级时间戳。""" + if _depth > 10: + return None + if isinstance(obj, dict): + value = obj.get("timestamp") + if isinstance(value, (int, float)) and value > 0: + return int(value) + provider = obj.get("providerData") + if isinstance(provider, dict): + value = provider.get("timestamp") + if isinstance(value, (int, float)) and value > 0: + return int(value) + for v in obj.values(): + r = _find_timestamp_ms(v, _depth + 1) + if r: + return r + elif isinstance(obj, list): + for v in obj: + r = _find_timestamp_ms(v, _depth + 1) + if r: + return r + return None + + +def _find_tool_records(obj: Any) -> list[dict[str, Any]]: + """从 transcript 记录里提取 tool 调用与 tool 结果片段。""" + out: list[dict[str, Any]] = [] + if not isinstance(obj, dict): + return out + + rec_type = str(obj.get("type") or "").strip().lower() + provider = obj.get("providerData") if isinstance(obj.get("providerData"), dict) else {} + if rec_type in {"function_call", "function_call_result"}: + tool_name = str(obj.get("name") or "").strip() + if tool_name: + record = { + "tool": tool_name, + "call_id": str(obj.get("callId") or "").strip() or None, + "kind": rec_type, + } + if rec_type == "function_call": + arguments = obj.get("arguments") + if isinstance(arguments, str) and arguments.strip(): + record["arguments"] = arguments + arguments_display_text = provider.get("argumentsDisplayText") + if isinstance(arguments_display_text, str) and arguments_display_text.strip(): + record["arguments_display_text"] = arguments_display_text.strip() + if rec_type == "function_call_result": + tool_result = provider.get("toolResult") + if isinstance(tool_result, dict): + content = tool_result.get("content") + if content is not None: + record["result_content"] = content + raw_response = tool_result.get("rawResponse") + if isinstance(raw_response, dict): + record["raw_response"] = raw_response + output = obj.get("output") + if isinstance(output, dict): + output_text = output.get("text") + if output_text is not None: + record["output_text"] = output_text + out.append(record) + + content = obj.get("content") + if isinstance(content, list): + for item in content: + if not isinstance(item, dict): + continue + if str(item.get("type") or "").strip().lower() != "tool_use": + continue + tool_name = str(item.get("name") or "").strip() + if not tool_name: + continue + out.append({ + "tool": tool_name, + "call_id": str(item.get("callId") or "").strip() or None, + "kind": "tool_use", + }) + + return out diff --git a/.cursor/skills/agent-observability/scripts/core/devflow.py b/.cursor/skills/agent-observability/scripts/core/devflow.py new file mode 100644 index 0000000..ab7ef79 --- /dev/null +++ b/.cursor/skills/agent-observability/scripts/core/devflow.py @@ -0,0 +1,234 @@ +"""Devflow 感知层（可选增强，不属于通用采集核心）。 + +这一层只在项目实际跑着 `.codebuddy/runtime` 描述的 multi-agents-devflow 工作流时才生效： +- 判断当前 session 是否属于某个 devflow team（`multi-agents-devflow-{task_slug}`） +- 读取该 team 对应的 `workflow-state.json`，投影成精简 stage 快照 +- 和上一次观测到的快照 diff，只把真正变化的 stage 产出为事件 + +任何解析失败（team 目录不存在 / workflow-state.json 缺失或损坏 / 字段缺失）都必须 +优雅降级为 None / 空列表，绝不能让不跑 devflow 的普通项目因为这一层报错。 +""" +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +from . import agent_identity, state as st + + +def task_slug_from_team_dir(team_dir: Path) -> str | None: + """从 devflow team 目录名里剥离出 task_slug（固定前缀 `multi-agents-devflow-`）。""" + name = team_dir.name + prefix = agent_identity.TEAM_PREFIX + if not name.startswith(prefix): + return None + slug = name[len(prefix):].strip() + return slug or None + + +def resolve_devflow_context(cwd: str, sid: str, cached_team_dir: str | None = None) -> dict[str, Any] | None: + """判断当前 session 是否处于某次 devflow 运行中；不是则返回 None。 + + 优先复用 `agent_identity.resolve_team_dir` 做 team 发现（Classic 全部场景，以及 + Portable 在 `topology: team` 宿主——目前是 CodeBuddy——下也走同一套 + `multi-agents-devflow-{task_slug}` 命名，可以直接复用，不用区分 edition）。 + + `topology: spawn` 的宿主（Codex/Claude/Cursor 的 Portable 适配器）不创建 team + 目录，找不到时退化成 `_scan_artifacts_for_active_run` 直接扫 `artifacts/` 目录。 + 这个兜底本身就是尽力而为的启发式，不保证唯一/精确，见该函数的说明。 + + `artifacts_dir` 按 `devflow.defaults.yaml` 的默认值 `{project_root}/artifacts/{task_slug}` + 推算——项目若覆写了 `artifacts.root_dir`，这里暂不感知，读取 workflow-state.json + 找不到文件会安全返回 None，不会误报。 + """ + team_dir = agent_identity.resolve_team_dir(cwd, sid, cached_team_dir) + if team_dir is not None: + task_slug = task_slug_from_team_dir(team_dir) + if task_slug: + artifacts_dir = Path(cwd).expanduser() / "artifacts" / task_slug + workflow_state_path = artifacts_dir / "workflow-state.json" + return { + "task_slug": task_slug, + "team_dir": str(team_dir), + "artifacts_dir": str(artifacts_dir), + "workflow_state_path": str(workflow_state_path), + } + return _scan_artifacts_for_active_run(cwd) + + +def _scan_artifacts_for_active_run(cwd: str) -> dict[str, Any] | None: + """没有 team 目录时的兜底发现：直接扫 `{cwd}/artifacts/*/workflow-state.json`。 + + 用于 `topology: spawn` 的宿主（没有 `.codebuddy/teams/` 这类目录可以反查），以及 + Classic 没有真正暴露 `team_create` 时的降级场景。这是启发式，不是精确匹配： + 多个 task_slug 存在时，优先选"还没跑完"的那个；都跑完或都没跑完时选 + `workflow-state.json` 文件 mtime 最新的一个。"跑完"的判定要兼容两套 schema—— + Classic（v1.3）没有顶层 `status` 字段，看 `last_event == "workflow_completed"`； + Portable（v2.0）看顶层 `status in {"completed", "failed"}`。项目里如果同时有多个 + 真正并发、都还没跑完的 devflow 运行，这个兜底可能选错——已知限制，不在这次范围内解决。 + """ + artifacts_root = Path(cwd).expanduser() / "artifacts" + if not artifacts_root.is_dir(): + return None + candidates: list[tuple[tuple[int, float], str, Path]] = [] + try: + children = list(artifacts_root.iterdir()) + except Exception: + return None + for child in children: + if not child.is_dir(): + continue + state_path = child / "workflow-state.json" + state = read_workflow_state(str(state_path)) + if not isinstance(state, dict) or not isinstance(state.get("stages"), dict): + continue + try: + mtime = state_path.stat().st_mtime + except Exception: + mtime = 0.0 + finished = ( + str(state.get("last_event") or "") == "workflow_completed" + or str(state.get("status") or "") in {"completed", "failed"} + ) + not_finished = 0 if finished else 1 + candidates.append(((not_finished, mtime), child.name, state_path)) + if not candidates: + return None + candidates.sort(key=lambda item: item[0], reverse=True) + _, task_slug, state_path = candidates[0] + return { + "task_slug": task_slug, + "team_dir": None, + "artifacts_dir": str(state_path.parent), + "workflow_state_path": str(state_path), + } + + +def read_workflow_state(path: str) -> dict[str, Any] | None: + """安全读取 workflow-state.json；文件不存在或解析失败都返回 None。""" + try: + p = Path(path) + if not p.is_file(): + return None + data = json.loads(p.read_text("utf-8")) + return data if isinstance(data, dict) else None + except Exception: + return None + + +def stage_snapshot(workflow_state: dict[str, Any] | None) -> dict[str, Any]: + """把 workflow-state.json 投影成精简快照，供比对和事件输出使用。 + + 兼容两套 schema：Classic（v1.3，字段名 `executor`，有 `review_result`）和 + Portable（v2.0，字段名 `executor_role`，没有 `review_result`，但顶层多了 + `execution_mode`/`host_adapter`/`run_id`/`team_name` 这些 Classic 没有的上下文）。 + 按 `version` 字段区分，取不到就都尝试取一遍，不强制要求调用方先判断是哪个 edition。 + """ + if not isinstance(workflow_state, dict): + return {} + stages_raw = workflow_state.get("stages") + stages: dict[str, Any] = {} + if isinstance(stages_raw, dict): + for name, info in stages_raw.items(): + if not isinstance(info, dict): + continue + stages[name] = { + "status": info.get("status"), + "executor": info.get("executor") or info.get("executor_role"), + "retry_count": info.get("retry_count", 0), + "review_result": info.get("review_result"), + } + return { + "current_stage": workflow_state.get("current_stage"), + "size_class": workflow_state.get("size_class"), + "run_mode": workflow_state.get("run_mode"), + "schema_version": workflow_state.get("version"), + "execution_mode": workflow_state.get("execution_mode"), + "host_adapter": workflow_state.get("host_adapter"), + "run_id": workflow_state.get("run_id"), + "stages": stages, + } + + +def diff_stage_changes(prev: dict[str, Any] | None, curr: dict[str, Any]) -> list[dict[str, Any]]: + """比较两次 stage 快照，只返回 status/retry_count/review_result 真正变化的阶段。 + + 首次观测（`prev` 为 None，即这个 session 第一次检测到 devflow）不产出任何变更—— + 避免刚接入 observability 时，把一个已经跑了大半的 devflow 运行的全部历史阶段 + 当成"新事件"一次性炸出来。之后每次变化都会被正常捕获。 + """ + if not isinstance(curr, dict) or prev is None: + return [] + curr_stages = curr.get("stages") or {} + prev_stages = prev.get("stages") if isinstance(prev, dict) else {} + if not isinstance(prev_stages, dict): + prev_stages = {} + changes: list[dict[str, Any]] = [] + for name, info in curr_stages.items(): + if not isinstance(info, dict): + continue + before = prev_stages.get(name) + before = before if isinstance(before, dict) else {} + fields = ("status", "retry_count", "review_result") + if any(before.get(f) != info.get(f) for f in fields): + changes.append({ + "stage": name, + "status": info.get("status"), + "executor": info.get("executor"), + "retry_count": info.get("retry_count", 0), + "review_result": info.get("review_result"), + }) + return changes + + +def resolve_and_diff(state_path: Path, sid: str, cwd: str) -> dict[str, Any] | None: + """解析当前 session 归属的 devflow 上下文，返回上下文 + 本次观测到的 stage 变更。 + + **不缓存"当前归属哪个 task_slug"这个结论本身**——同一个 sid 在其生命周期内可能 + 先后归属不同的 devflow 运行。这不是理论场景：当运行时没有暴露真正的 + `team_create`/`send_message`（因此没有独立的 team 成员 sid），devflow 会退化成 + 在同一个长生命周期 session 里，先后跑好几次 `/start-devflow`；每次都必须重新判定 + "现在最合适的 task_slug 是哪个"，而不能沿用第一次探测到的那个——沿用旧结论会把 + 第二次运行的所有事件都错误地归到第一次的 task_slug 下。 + + 重新判定的代价很低：`resolve_devflow_context` 对团队目录的探测会用上一次找到的 + `team_dir` 做快速校验（只在真正失效时才重新扫描 `.codebuddy/teams/`）； + `artifacts/` 兜底扫描也只是一次浅层 `iterdir()`，不是全量遍历。 + + 真正跨调用持久化的只有**按 task_slug 分别保存的 stage 快照历史**——切换到另一个 + task_slug 不会污染对方的 diff 基线，也不会因为切回旧 task_slug 而把它已经观测过的 + 历史重新当成"首次观测"。 + + 返回 None 表示这次调用没有探测到任何 devflow 上下文；否则返回 + `{task_slug, artifacts_dir, workflow_state_path, current_stage, size_class, changes}`。 + """ + + def _update(state: dict[str, Any]) -> dict[str, Any] | None: + dv = st.get_devflow_state(state, sid) + cached_team_dir = (dv.get("context") or {}).get("team_dir") if isinstance(dv.get("context"), dict) else None + context = resolve_devflow_context(cwd, sid, cached_team_dir) + if not isinstance(context, dict): + st.set_devflow_state(state, sid, {"context": None}) + return None + + task_slug = str(context.get("task_slug") or "") + workflow_state = read_workflow_state(context.get("workflow_state_path", "")) + curr = stage_snapshot(workflow_state) + snapshots = dv.get("snapshots") + if not isinstance(snapshots, dict): + snapshots = {} + prev = snapshots.get(task_slug) + changes = diff_stage_changes(prev if isinstance(prev, dict) else None, curr) + snapshots[task_slug] = curr + st.set_devflow_state(state, sid, {"context": context, "snapshots": snapshots}) + + result = dict(context) + result["current_stage"] = curr.get("current_stage") + result["size_class"] = curr.get("size_class") + result["schema_version"] = curr.get("schema_version") + result["execution_mode"] = curr.get("execution_mode") + result["changes"] = changes + return result + + return st.update_state_locked(state_path, _update) diff --git a/.cursor/skills/agent-observability/scripts/core/emitter.py b/.cursor/skills/agent-observability/scripts/core/emitter.py new file mode 100644 index 0000000..a41dfc0 --- /dev/null +++ b/.cursor/skills/agent-observability/scripts/core/emitter.py @@ -0,0 +1,578 @@ +"""ndjson 发射层。 + +只输出当前真正需要的核心字段： +- common: data_source +- tool: event, sid, ts, agent, tool, ms, skill, rule, transcript_path +- usage: event, sid, ts, agent, tool, tokens, model, cost_usd, transcript_path +- start: event, sid, ts, agent +- stop: event, sid, ts, agent, tokens, model, cost_usd, cost_session_usd, transcript_path +- error: event, sid, ts, phase, error + +历史 x_* 扩展字段、stop skill/rule 汇总、schema 版本号等都不再生成。 + +`cost_usd` 的价格表 = 内置 `config/pricing.json` 与用户覆盖文件（默认 +`/.codebuddy/agent-observability/pricing.overrides.json`，可用 +`AOBS_PRICING_OVERRIDES_PATH` 改路径）的字段级合并结果，详见 `load_prices()`。 +""" +from __future__ import annotations + +import json +import os +import time +from functools import lru_cache +from pathlib import Path +from typing import Any + +from . import cls_sink +from . import state as st + +DEFAULT_PRICES_PATH = Path(__file__).resolve().parents[2] / "config" / "pricing.json" +DEFAULT_DATA_SOURCE = "codebuddy-cli" + + +def _default_overrides_path() -> Path | None: + """用户定价覆盖文件的默认位置：`/.codebuddy/agent-observability/pricing.overrides.json`。 + + 刻意放在 `skills/` 树**外**：`scripts/build-classic-hosts.py` 会整棵同步 + `.codebuddy/skills`，放树内会 (a) 每次改动都产生 `--check` drift，(b) 把用户的 + 自定义价格复制进 `.claude`/`.cursor` 宿主包；放树外则既不参与同步，也不会被 + `config/pricing.json` 的后续更新冲掉。 + """ + try: + project_root = Path(__file__).resolve().parents[5] + except IndexError: + return None + return project_root / ".codebuddy" / "agent-observability" / "pricing.overrides.json" + + +# None 表示"拿不到项目根"，此时等同于"没有覆盖文件"（纯内置价格表）。 +DEFAULT_OVERRIDES_PATH: Path | None = _default_overrides_path() + + +def get_log_path(base_dir: Path) -> Path: + """返回 metrics 日志路径，并确保目录已存在。""" + log_dir = base_dir / "logs" + log_dir.mkdir(parents=True, exist_ok=True) + return log_dir / "metrics.ndjson" + + +def _first_record_ts(log_path: Path, sid: str) -> float | None: + """从 metrics.ndjson 中找到某个 session 的最早 ts。 + + state 只保留最近几个 session，长流程或历史 session 可能被 prune 后重建， + 因此 `started_at` 不能作为唯一权威。日志里的首条 ts 更接近 + session/workflow 的真实起点。 + """ + if not sid or not log_path.exists(): + return None + first_ts: float | None = None + try: + with log_path.open("r", encoding="utf-8") as fp: + for line in fp: + if not line.strip().startswith("{"): + continue + try: + rec = json.loads(line) + except Exception: + continue + if rec.get("sid") != sid: + continue + try: + ts = float(rec.get("ts")) + except Exception: + continue + if first_ts is None or ts < first_ts: + first_ts = ts + except Exception: + return None + return first_ts + + +def _inject_session_duration(record: dict[str, Any], log_path: Path) -> None: + """从 state 文件读取 session started_at，计算 session_duration_sec 注入 record。 + 同时扫描 metrics.ndjson 累计 session 级别 token 总量。""" + sid = record.get("sid") + if not sid or "session_duration_sec" in record: + return + try: + now = float(record.get("ts") or time.time()) + started_candidates: list[float] = [] + state_path = log_path.parent / ".state.json" + state = st.load_state(state_path) + sess = state.get(sid) + if isinstance(sess, dict): + try: + state_started_at = float(sess.get("started_at") or 0) + except Exception: + state_started_at = 0.0 + if state_started_at > 0: + started_candidates.append(state_started_at) + log_started_at = _first_record_ts(log_path, str(sid)) + if log_started_at is not None and log_started_at > 0: + started_candidates.append(log_started_at) + if not started_candidates: + started_candidates.append(now) + started_at = min(started_candidates) + record["session_duration_sec"] = round(max(0.0, now - started_at), 3) + except Exception: + return + + # 累计 session 级别 token（扫描 metrics.ndjson 尾部） + try: + total_input = 0 + total_output = 0 + total_cache = 0 + if log_path.exists(): + with log_path.open("r", encoding="utf-8") as fp: + for line in fp: + if not line.strip().startswith("{"): + continue + try: + rec = json.loads(line) + except Exception: + continue + if rec.get("sid") != sid or rec.get("event") != "usage": + continue + tokens = rec.get("tokens") + if not isinstance(tokens, dict): + continue + total_input += int(tokens.get("input") or 0) + total_output += int(tokens.get("output") or 0) + total_cache += int(tokens.get("cache_read") or 0) + record["session_total_tokens"] = total_input + total_output + record["session_input_tokens"] = total_input + record["session_output_tokens"] = total_output + record["session_cache_tokens"] = total_cache + except Exception: + return + + +def emit(log_path: Path, record: dict[str, Any]) -> None: + """向 ndjson 日志追加一条记录。""" + record.setdefault("data_source", DEFAULT_DATA_SOURCE) + record.setdefault("ts", time.time()) + _inject_session_duration(record, log_path) + with log_path.open("a", encoding="utf-8") as fp: + fp.write(json.dumps(record, ensure_ascii=False) + "\n") + try: + cls_sink.mirror_record(record) + except Exception: + return + + +def build_tool_event( + sid: str, + tool: str | None, + duration_ms: int | None, + active_agent: str | None = None, + transcript_path: str | None = None, + turn_id: str | None = None, + task_slug: str | None = None, + stage: str | None = None, + pt_id: str | None = None, +) -> dict[str, Any]: + """构造 tool 事件记录，只包含调用信息与耗时。 + + `task_slug` / `stage` / `pt_id` 只有在检测到 devflow 运行时才会出现（见 + `core/devflow.py`），非 devflow 项目不受影响。 + """ + record = { + "event": "tool", + "sid": sid, + "agent": active_agent or "main", + "tool": tool, + "ms": duration_ms, + } + if transcript_path: + record["transcript_path"] = transcript_path + if turn_id: + record["turn_id"] = turn_id + if task_slug: + record["task_slug"] = task_slug + if stage: + record["stage"] = stage + if pt_id: + record["pt_id"] = pt_id + return record + + +def build_usage_event( + sid: str, + tool: str | None, + tokens: dict[str, Any], + active_agent: str | None = None, + model: str | None = None, + cost_usd: float | None = None, + transcript_path: str | None = None, + source_offset: int | None = None, + turn_id: str | None = None, + message_id: str | None = None, + task_slug: str | None = None, + stage: str | None = None, +) -> dict[str, Any]: + """构造 LLM usage 事件记录。""" + record: dict[str, Any] = { + "event": "usage", + "sid": sid, + "agent": active_agent or "main", + "tool": tool, + "tokens": tokens, + } + if model: + record["model"] = model + if cost_usd is not None: + record["cost_usd"] = cost_usd + if transcript_path: + record["transcript_path"] = transcript_path + if source_offset is not None: + record["source_offset"] = source_offset + if turn_id: + record["turn_id"] = turn_id + if message_id: + record["message_id"] = message_id + if task_slug: + record["task_slug"] = task_slug + if stage: + record["stage"] = stage + return record + + +def build_start_event( + sid: str, + agent: str | None = None, +) -> dict[str, Any]: + """构造最小化的 start 事件记录。 + + 初始化阶段没有明确角色时，统一记为 ``main``。 + """ + return { + "event": "start", + "sid": sid, + "agent": agent or "main", + } + + +def build_prompt_submit_event( + sid: str, + turn_id: str, + agent: str | None = None, + prompt_len: int | None = None, +) -> dict[str, Any]: + """构造 `user_prompt_submit` 事件，作为 turn 边界标记。 + + 这条记录会落到 metrics.ndjson，方便下游把 turn 和 AgentLens trace 对上。 + """ + record: dict[str, Any] = { + "event": "user_prompt_submit", + "sid": sid, + "agent": agent or "main", + "turn_id": turn_id, + } + if prompt_len is not None: + record["prompt_len"] = int(prompt_len) + return record + + +def build_stop_event( + sid: str, + tokens: dict | None = None, + model: str | None = None, + cost_usd: float | None = None, + cost_session_usd: float | None = None, + active_agent: str | None = None, + transcript_path: str | None = None, + source_offset: int | None = None, + turn_id: str | None = None, + message_id: str | None = None, + task_slug: str | None = None, + stage: str | None = None, +) -> dict[str, Any]: + """构造最小化的 stop 事件记录。""" + record: dict[str, Any] = { + "event": "stop", + "sid": sid, + "agent": active_agent or "main", + } + if tokens: + record["tokens"] = tokens + if model: + record["model"] = model + if cost_usd is not None: + record["cost_usd"] = cost_usd + if cost_session_usd is not None: + record["cost_session_usd"] = cost_session_usd + if transcript_path: + record["transcript_path"] = transcript_path + if source_offset is not None: + record["source_offset"] = source_offset + if turn_id: + record["turn_id"] = turn_id + if message_id: + record["message_id"] = message_id + if task_slug: + record["task_slug"] = task_slug + if stage: + record["stage"] = stage + return record + + +def build_error_event( + phase: str, + error: str, + sid: str | None = None, +) -> dict[str, Any]: + """构造最小化的 error 事件记录。""" + return { + "event": "error", + "sid": sid or "", + "phase": phase, + "error": error, + } + + +def build_stage_transition_event( + sid: str, + task_slug: str, + stage: str, + status: str | None = None, + executor: str | None = None, + retry_count: int = 0, + review_result: str | None = None, +) -> dict[str, Any]: + """构造 devflow stage 变更事件（只在检测到 devflow 运行、且 stage 真的变化时发出）。 + + 对应 `workflow-state.json` 里某个 stage 的 `status`/`retry_count`/`review_result` + 发生变化——retry_count 上升或 review_result="failed" 是 devflow 里最值得关注的 + 信号（返工/打回循环），比 hook 自身的 `error` 事件更贴近业务语义。 + """ + record: dict[str, Any] = { + "event": "stage_transition", + "sid": sid, + "task_slug": task_slug, + "stage": stage, + } + if status: + record["status"] = status + if executor: + record["executor"] = executor + if retry_count: + record["retry_count"] = int(retry_count) + if review_result: + record["review_result"] = review_result + return record + + +def resolve_overrides_path() -> Path | None: + """返回当前生效的覆盖文件路径：`AOBS_PRICING_OVERRIDES_PATH` 优先，否则默认路径。 + + 返回 `None` 表示"不加载任何覆盖"（纯内置价格表）。支持 `~` 展开。 + """ + raw = os.environ.get("AOBS_PRICING_OVERRIDES_PATH", "").strip() + if raw: + return Path(raw).expanduser() + return DEFAULT_OVERRIDES_PATH + + +def _coerce_price_table(data: Any) -> dict[str, dict[str, float]]: + """把任意 JSON 结果规整成 `{model: {field: float}}`，坏数据按字段/按模型跳过。 + + 与内置表加载的差别只有一处：内置表遇到非数字会整体抛错降级，这里逐字段跳过—— + 用户的补丁文件里写错一个字段不应该把其余正确的覆盖一起丢掉，更不应该让 hook 挂掉。 + """ + if not isinstance(data, dict): + return {} + table: dict[str, dict[str, float]] = {} + for raw_key, raw_value in data.items(): + if not isinstance(raw_value, dict): + continue + key = str(raw_key).strip().lower() + if not key: + continue + row: dict[str, float] = {} + for price_key, price_value in raw_value.items(): + # bool 是 int 的子类，True 会被 float() 变成 1.0，这里按"不是价格"处理。 + if price_value is None or isinstance(price_value, bool): + continue + try: + row[str(price_key).strip().lower()] = float(price_value) + except (TypeError, ValueError): + continue + # 全是坏字段的模型不落表：否则它会被 `lookup_price` 命中成 {}， + # 既算不出成本又不算"未定价"，看板上会变成一个查不到原因的空洞。 + if row: + table[key] = row + return table + + +def load_price_overrides(path: Path | None = None) -> dict[str, dict[str, float]]: + """加载用户定价覆盖文件；任何异常都静默降级为空覆盖（纯内置价格表）。 + + 静默是刻意的：这个文件是给人手写的常驻配置，hook 每次都是新进程， + 一旦因格式问题抛异常/打印，会让整条 hook 链路变得不可用。 + """ + try: + target = path if path is not None else resolve_overrides_path() + if not target: + return {} + with Path(target).expanduser().open("r", encoding="utf-8") as fp: + data = json.load(fp) + except Exception: + return {} + return _coerce_price_table(data) + + +def merge_prices( + base: dict[str, dict[str, float]] | None, + overrides: dict[str, dict[str, float]] | None, +) -> dict[str, dict[str, float]]: + """字段级合并价格表：覆盖里写出的字段替换内置值，没写的字段沿用内置值。 + + 可以新增内置表里不存在的模型；**不支持删除**内置模型或字段（合并只能加不能减）。 + """ + merged: dict[str, dict[str, float]] = {} + for model, row in (base or {}).items(): + if isinstance(row, dict): + merged[model] = dict(row) + for model, patch in (overrides or {}).items(): + if not isinstance(patch, dict): + continue + merged.setdefault(model, {}).update(patch) + return merged + + +def _load_builtin_prices() -> dict[str, dict[str, float]]: + """从 `config/pricing.json` 中一次性加载内置模型价格配置（不含用户覆盖）。""" + path = os.environ.get("AOBS_PRICES_PATH", "").strip() + prices_path = Path(path).expanduser() if path else DEFAULT_PRICES_PATH + try: + with prices_path.open("r", encoding="utf-8") as fp: + data = json.load(fp) + except Exception: + return {} + if not isinstance(data, dict): + return {} + return { + str(key).lower(): {price_key: float(price_value) for price_key, price_value in value.items()} + for key, value in data.items() + if isinstance(value, dict) + } + + +@lru_cache(maxsize=1) +def load_prices() -> dict[str, dict[str, float]]: + """内置价格表 + 用户覆盖文件的合并结果（字段级）。 + + 结果带 `lru_cache`：hook 是短命进程，一次会话里只该读一次盘；长驻进程或单测里 + 改了覆盖文件/环境变量后，调 `clear_price_cache()` 让下一次调用重新加载。 + """ + return merge_prices(_load_builtin_prices(), load_price_overrides()) + + +def clear_price_cache() -> None: + """丢弃 `load_prices()` 的缓存，让下一次调用重读磁盘与环境变量。""" + load_prices.cache_clear() + + +def lookup_price(model: str | None) -> dict[str, float] | None: + """为一个具体模型名找到最匹配的价格配置项。""" + if not model or not isinstance(model, str): + return None + normalized = model.lower().strip() + if not normalized: + return None + prices = load_prices() + if normalized in prices: + return prices[normalized] + best_key: str | None = None + for key in prices: + if key in normalized and (best_key is None or len(key) > len(best_key)): + best_key = key + if best_key is None: + return None + return prices[best_key] + + +def is_unpriced(model: str | None) -> bool: + """模型是否完全没命中价格表（含模糊匹配）。 + + "未定价"的唯一口径就是 `lookup_price(...) is None`（D4）：模糊（最长子串）命中的 + 模型能算出成本，就不该出现在看板的"建议补价"提示里。 + """ + return lookup_price(model) is None + + +def estimate_cost(tokens: dict | None, model: str | None) -> dict[str, Any] | None: + """根据归一化后的 token 总量估算单条 usage 事件的成本。""" + if not isinstance(tokens, dict): + return None + rates = lookup_price(model) + if not rates: + return None + + def _int(key: str) -> int: + try: + return int(tokens.get(key) or 0) + except Exception: + return 0 + + output = _int("output") + cache_read = _int("cache_read") + cache_write = _int("cache_creation") + raw_input = _int("input") + fresh_input = raw_input - cache_read - cache_write if raw_input and raw_input >= (cache_read + cache_write) else raw_input + cost = ( + fresh_input * rates.get("input", 0.0) + + cache_read * rates.get("cache_read", 0.0) + + cache_write * rates.get("cache_write", 0.0) + + output * rates.get("output", 0.0) + ) / 1_000_000.0 + + matched_key = None + normalized = (model or "").lower() + for key in load_prices(): + if key in normalized and (matched_key is None or len(key) > len(matched_key)): + matched_key = key + if matched_key is None and normalized in load_prices(): + matched_key = normalized + + return { + "usd": round(cost, 6), + "model_matched": matched_key, + "rates": rates, + } + + +def cost_of(tokens: dict | None, model: str | None) -> float | None: + """只返回美元成本值的便捷封装。""" + if not tokens or not model: + return None + info = estimate_cost(tokens, model) + if not info: + return None + return info.get("usd") + + +def sum_session_cost(log_path: Path, sid: str) -> float | None: + """尽力根据已发出的事件反算整个 session 的总成本。""" + if not log_path.exists() or not sid: + return None + total = 0.0 + seen = False + try: + with log_path.open("r", encoding="utf-8") as fp: + for line in fp: + if not line.strip().startswith("{"): + continue + try: + rec = json.loads(line) + except Exception: + continue + if rec.get("sid") != sid or rec.get("event") not in {"usage", "tool"}: + continue + cost = rec.get("cost_usd") + if isinstance(cost, (int, float)): + total += float(cost) + seen = True + except Exception: + return None + return round(total, 6) if seen else None diff --git a/.cursor/skills/agent-observability/scripts/core/runtime.py b/.cursor/skills/agent-observability/scripts/core/runtime.py new file mode 100644 index 0000000..9c647bc --- /dev/null +++ b/.cursor/skills/agent-observability/scripts/core/runtime.py @@ -0,0 +1,598 @@ +from __future__ import annotations +"""Hook 主编排入口。 + +这一层负责把外部 hook phase 分发到内部能力： +- session-start / user-prompt-submit / pre / post / stop +- 组装统一运行时路径 +- 串联 collector / emitter / agentlens，并承担 transcript 运行时编排 +""" + +import os +import re +import time +import traceback +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from . import agent_identity, agentlens, collector, devflow, emitter, scanner, state as st + + +@dataclass(frozen=True) +class RuntimePaths: + """运行时常用路径集合，避免每个分支重复计算。""" + base_dir: Path + state_path: Path + pending_path: Path + log_path: Path + + +def base_dir() -> Path: + """返回 hook 运行时使用的根目录。""" + # 运行时产物统一写到 skill 根目录下的 logs/，保持与旧路径兼容。 + return Path(__file__).resolve().parents[2] + + +def build_runtime_paths() -> RuntimePaths: + """构造本次 hook 运行要用到的路径集合。""" + base = base_dir() + return RuntimePaths( + base_dir=base, + state_path=st.get_state_path(base), + pending_path=st.get_pending_path(base), + log_path=emitter.get_log_path(base), + ) + + +def session_id_of(data: dict[str, Any]) -> str: + """兼容不同字段名，提取 session id。""" + return str(data.get("session_id") or data.get("sid") or "") + + +def cwd(data: dict[str, Any]) -> str: + """解析当前项目工作目录。""" + return str(data.get("cwd") or os.environ.get("CODEBUDDY_PROJECT_DIR") or os.getcwd()) + + +def normalize_phase(raw: str | None) -> str: + """把多种 phase 写法折叠成统一内部枚举。""" + phase = (raw or "").strip().lower() + return { + "sessionstart": "session-start", + "session_start": "session-start", + "session-start": "session-start", + "pretooluse": "pre", + "pre": "pre", + "posttooluse": "post", + "post": "post", + "stop": "stop", + "userpromptsubmit": "user-prompt-submit", + "user_prompt_submit": "user-prompt-submit", + "user-prompt-submit": "user-prompt-submit", + "prompt": "user-prompt-submit", + }.get(phase, phase) + + +def emit_error(phase: str, sid: str, err: Exception) -> None: + """把运行时异常写成 error 事件，而不是直接中断 hook。""" + rec = emitter.build_error_event(phase=phase, error=f"{type(err).__name__}: {err}", sid=sid) + rec["x_traceback"] = traceback.format_exc(limit=5)[-1200:] + emitter.emit(build_runtime_paths().log_path, rec) + + +def derive_turn_id(data: dict[str, Any]) -> str: + """优先复用外部已有 id，否则生成一个本地 turn id。""" + for key in ("turn_id", "prompt_id", "request_id", "message_id"): + raw = data.get(key) + if isinstance(raw, str) and raw.strip(): + return raw.strip() + if isinstance(raw, (int, float)) and raw: + return str(raw) + return f"turn-{int(time.time() * 1000)}" + + +def prompt_text_length(data: dict[str, Any]) -> int | None: + """统计用户 prompt 的文本长度，用于 turn 边界指标。""" + for key in ("prompt", "user_prompt", "message", "content", "text"): + val = data.get(key) + if isinstance(val, str): + return len(val) + if isinstance(val, list): + total = 0 + for item in val: + if isinstance(item, dict): + txt = item.get("text") + if isinstance(txt, str): + total += len(txt) + elif isinstance(item, str): + total += len(item) + if total: + return total + return None + + +def extract_prompt_text(data: dict[str, Any]) -> str: + """从 hook payload 中提取可读的 prompt 文本。""" + for key in ("prompt", "user_prompt", "message", "text"): + val = data.get(key) + if isinstance(val, str) and val.strip(): + return val.strip() + if isinstance(val, list): + parts = [] + for item in val: + if isinstance(item, dict): + txt = item.get("text") + if isinstance(txt, str): + parts.append(txt) + elif isinstance(item, str): + parts.append(item) + if parts: + return "\n".join(parts).strip() + return "" + + +def derive_business_scenario(prompt_text: str) -> str | None: + """从 prompt 文本中派生一个简短业务场景标识。""" + if not prompt_text: + return None + + m = re.search(r"标题[：:]\s*(.+?)(?:\s*描述[：:]|\s*$)", prompt_text, re.DOTALL) + if m: + title = m.group(1).strip()[:60] + else: + title = prompt_text[:50].strip() + + sanitized = re.sub(r"[^\w\u4e00-\u9fff\-]", "-", title) + sanitized = re.sub(r"-{2,}", "-", sanitized).strip("-") + return sanitized if sanitized else None + + +def handle_session_start(data: dict[str, Any]) -> None: + """处理 SessionStart：预热 inventory，写 start 事件，初始化 AgentLens。""" + current_sid = session_id_of(data) + if not current_sid: + return + paths = build_runtime_paths() + cwd_value = cwd(data) + skills_meta, rules_meta = scanner.scan_skills_and_rules(cwd_value) + collector.cache_inventory(paths.state_path, current_sid, skills_meta, rules_meta) + emitter.emit(paths.log_path, emitter.build_start_event(sid=current_sid, agent="main")) + agentlens.emit_session_start(state_path=paths.state_path, sid=current_sid, cwd=cwd_value, agent="main") + + +def handle_user_prompt_submit(data: dict[str, Any]) -> None: + """处理 UserPromptSubmit：建立 turn 边界，并打开 AgentLens turn。""" + current_sid = session_id_of(data) + if not current_sid: + return + paths = build_runtime_paths() + cwd_value = cwd(data) + turn_id = derive_turn_id(data) + prompt_len = prompt_text_length(data) + prompt_text = extract_prompt_text(data) + business_scenario = derive_business_scenario(prompt_text) + + emitter.emit( + paths.log_path, + emitter.build_prompt_submit_event(sid=current_sid, turn_id=turn_id, agent="main", prompt_len=prompt_len), + ) + agentlens.emit_turn_start( + state_path=paths.state_path, + sid=current_sid, + cwd=cwd_value, + turn_id=turn_id, + prompt_meta={"prompt_len": prompt_len} if prompt_len is not None else None, + business_scenario=business_scenario, + ) + + +def handle_pre(data: dict[str, Any]) -> None: + """处理 PreToolUse：仅记录 pending，等待 post 配对。""" + collector.record_pre(build_runtime_paths().pending_path, data) + + +def handle_post(data: dict[str, Any]) -> None: + """处理 PostToolUse：计算耗时、归因 agent、发 tool/usage 事件。""" + current_sid = session_id_of(data) + if not current_sid: + return + paths = build_runtime_paths() + cwd_value = cwd(data) + duration_ms = collector.record_post(paths.pending_path, data) + + skills_meta, rules_meta = collector.load_cached_inventory(paths.state_path, current_sid) + if not skills_meta and not rules_meta: + skills_meta, rules_meta = scanner.scan_skills_and_rules(cwd_value) + collector.cache_inventory(paths.state_path, current_sid, skills_meta, rules_meta) + + active_agent, _ = agent_identity.resolve_active_agent_for_event( + state_path=paths.state_path, + sid=current_sid, + cwd=cwd_value, + data=data, + ) + used_skills, used_rules = collector.record_tool_usage( + state_path=paths.state_path, + sid=current_sid, + data=data, + skills_meta=skills_meta, + rules_meta=rules_meta, + active_agent=active_agent, + collect_skills=True, + ) + + tool_name = str(data.get("tool_name") or "") + transcript_path = collector.transcript_path(data) + turn_id = current_turn_id(paths.state_path, current_sid) + event_ts = time.time() + tool_call_id = collector.extract_tool_call_id(data) + dv_ctx = resolve_devflow(paths.state_path, current_sid, cwd_value, paths.log_path) + task_slug = str(dv_ctx.get("task_slug") or "").strip() if dv_ctx else None + stage = str(dv_ctx.get("current_stage") or "").strip() if dv_ctx else None + flush_pending_tool_events( + state_path=paths.state_path, + log_path=paths.log_path, + sid=current_sid, + ) + tool_context = collector.find_current_tool_context( + state_path=paths.state_path, + sid=current_sid, + transcript_path=transcript_path, + tool_name=tool_name, + call_id=tool_call_id, + event_ts=event_ts, + claim=True, + ) + duplicate_tool_context = bool(isinstance(tool_context, dict) and tool_context.get("duplicate")) + if duplicate_tool_context: + tool_context = None + if isinstance(tool_context, dict): + resolved_transcript_path = str(tool_context.get("transcript_path") or "").strip() + if resolved_transcript_path: + transcript_path = resolved_transcript_path + resolved_agent = str(tool_context.get("agent") or "").strip() + if resolved_agent: + active_agent = resolved_agent + + tool_pt_id = str(tool_context.get("pt_id") or "").strip() if isinstance(tool_context, dict) else None + tool_event = emitter.build_tool_event( + sid=current_sid, + tool=tool_name, + duration_ms=duration_ms, + active_agent=active_agent, + transcript_path=transcript_path, + turn_id=turn_id, + task_slug=task_slug, + stage=stage, + pt_id=tool_pt_id, + ) + tool_event["ts"] = event_ts + tool_event["skill"] = used_skills + tool_event["rule"] = used_rules + tool_event["cwd"] = cwd_value + if tool_call_id: + tool_event["call_id"] = tool_call_id + if isinstance(tool_context, dict): + tool_message_id = str(tool_context.get("message_id") or "").strip() or None + if tool_message_id: + tool_event["message_id"] = tool_message_id + tool_details = tool_context.get("tool_details") + if isinstance(tool_details, dict) and tool_details: + tool_event["tool_details"] = dict(tool_details) + + usage_events = emit_transcript_events( + state_path=paths.state_path, + log_path=paths.log_path, + sid=current_sid, + transcript_paths=collector.related_transcript_paths(current_sid, transcript_path), + current_transcript_path=transcript_path, + active_agent=active_agent, + tool_name=tool_name, + event_kind="usage", + max_lines=80, + turn_id=turn_id, + task_slug=task_slug, + stage=stage, + ) + if duplicate_tool_context: + if not usage_events: + return + agentlens.emit_post_step( + state_path=paths.state_path, + sid=current_sid, + tool_event=None, + usage_events=usage_events, + ) + return + if str(tool_event.get("message_id") or "").strip(): + emitter.emit(paths.log_path, tool_event) + agentlens.emit_post_step( + state_path=paths.state_path, + sid=current_sid, + tool_event=tool_event, + usage_events=usage_events, + ) + return + + fallback_usage = collector.find_fallback_usage_event(tool_event, usage_events) + fallback_usage_mid = str((fallback_usage or {}).get("message_id") or "").strip() or None + if fallback_usage_mid: + tool_event["message_id"] = fallback_usage_mid + emitter.emit(paths.log_path, tool_event) + agentlens.emit_post_step( + state_path=paths.state_path, + sid=current_sid, + tool_event=tool_event, + usage_events=usage_events, + ) + return + + st.append_pending_tool_emit(paths.state_path, current_sid, tool_event) + + +def handle_stop(data: dict[str, Any]) -> None: + """处理 Stop：补发 transcript 事件、汇总 session 成本并关闭 AgentLens。""" + current_sid = session_id_of(data) + if not current_sid: + return + paths = build_runtime_paths() + cwd_value = cwd(data) + active_agent, _ = agent_identity.resolve_active_agent_for_event( + state_path=paths.state_path, + sid=current_sid, + cwd=cwd_value, + data=data, + ) + transcript_path = collector.transcript_path(data) + turn_id = current_turn_id(paths.state_path, current_sid) + dv_ctx = resolve_devflow(paths.state_path, current_sid, cwd_value, paths.log_path) + task_slug = str(dv_ctx.get("task_slug") or "").strip() if dv_ctx else None + stage = str(dv_ctx.get("current_stage") or "").strip() if dv_ctx else None + stop_events = emit_transcript_events( + state_path=paths.state_path, + log_path=paths.log_path, + sid=current_sid, + transcript_paths=collector.settled_related_transcript_paths(current_sid, transcript_path), + current_transcript_path=transcript_path, + active_agent=active_agent, + tool_name="__stop__", + event_kind="stop", + max_lines=200, + cost_session=emitter.sum_session_cost(paths.log_path, current_sid), + turn_id=turn_id, + task_slug=task_slug, + stage=stage, + ) + flush_pending_tool_events( + state_path=paths.state_path, + log_path=paths.log_path, + sid=current_sid, + force_emit_fallback=True, + ) + agentlens.emit_session_stop( + state_path=paths.state_path, + sid=current_sid, + cwd=cwd_value, + active_agent=active_agent, + transcript_path=transcript_path, + stop_events=stop_events, + ) + + +def resolve_devflow(state_path: Path, sid: str, cwd_value: str, log_path: Path) -> dict[str, Any] | None: + """解析 devflow 上下文，并把本次观测到的 stage 变更立即发成 `stage_transition` 事件。 + + 任何异常都吞掉、返回 None——devflow 感知是可选增强，绝不能让不跑 devflow 的 + 项目或 workflow-state.json 格式变化导致 hook 报错。 + """ + try: + dv_ctx = devflow.resolve_and_diff(state_path, sid, cwd_value) + except Exception: + return None + if not isinstance(dv_ctx, dict): + return None + task_slug = str(dv_ctx.get("task_slug") or "").strip() + if not task_slug: + return None + for change in dv_ctx.get("changes") or []: + if not isinstance(change, dict): + continue + event = emitter.build_stage_transition_event( + sid=sid, + task_slug=task_slug, + stage=str(change.get("stage") or ""), + status=change.get("status"), + executor=change.get("executor"), + retry_count=int(change.get("retry_count") or 0), + review_result=change.get("review_result"), + ) + emitter.emit(log_path, event) + return dv_ctx + + +def current_turn_id(state_path: Path, sid: str) -> str | None: + """从 state 中读取当前 session 激活中的 turn id。""" + try: + turn = st.get_current_turn(state_path, sid) + except Exception: + turn = None + if isinstance(turn, dict): + raw = turn.get("turn_id") + if isinstance(raw, str) and raw.strip(): + return raw.strip() + return None + + +def flush_pending_tool_events( + *, + state_path: Path, + log_path: Path, + sid: str, + force_emit_fallback: bool = False, +) -> None: + """强制 flush 延迟的 tool 事件，常用于 stop/replay 路径。""" + pending = st.get_pending_tool_emits(state_path, sid) + if not pending: + return + + remaining: list[dict[str, Any]] = [] + for tool_event in pending: + tool_name = str(tool_event.get("tool") or "").strip() + transcript_path = str(tool_event.get("transcript_path") or "").strip() + call_id = str(tool_event.get("call_id") or "").strip() or None + event_ts = tool_event.get("ts") + try: + event_ts_float = float(event_ts) if event_ts is not None else None + except Exception: + event_ts_float = None + + tool_context = collector.find_current_tool_context( + state_path=state_path, + sid=sid, + transcript_path=transcript_path, + tool_name=tool_name, + call_id=call_id, + event_ts=event_ts_float, + claim=True, + ) + if isinstance(tool_context, dict) and tool_context.get("duplicate"): + continue + if isinstance(tool_context, dict): + resolved_transcript_path = str(tool_context.get("transcript_path") or "").strip() + if resolved_transcript_path: + tool_event["transcript_path"] = resolved_transcript_path + resolved_agent = str(tool_context.get("agent") or "").strip() + if resolved_agent: + tool_event["agent"] = resolved_agent + tool_message_id = str(tool_context.get("message_id") or "").strip() or None + if tool_message_id: + tool_event["message_id"] = tool_message_id + tool_details = tool_context.get("tool_details") + if isinstance(tool_details, dict) and tool_details: + tool_event["tool_details"] = dict(tool_details) + emitter.emit(log_path, tool_event) + agentlens.emit_post_step( + state_path=state_path, + sid=sid, + tool_event=tool_event, + usage_events=[], + ) + continue + if force_emit_fallback: + emitter.emit(log_path, tool_event) + agentlens.emit_post_step( + state_path=state_path, + sid=sid, + tool_event=tool_event, + usage_events=[], + ) + continue + remaining.append(tool_event) + + st.replace_pending_tool_emits(state_path, sid, remaining) + + +def emit_transcript_events( + *, + state_path: Path, + log_path: Path, + sid: str, + transcript_paths: list[str], + current_transcript_path: str, + active_agent: str, + tool_name: str, + event_kind: str, + max_lines: int, + cost_session: float | None = None, + turn_id: str | None = None, + task_slug: str | None = None, + stage: str | None = None, +) -> list[dict[str, Any]]: + """发出基于 transcript 增量重建得到的 usage/stop 事件。""" + emitted_events: list[dict[str, Any]] = [] + current_resolved = str(Path(current_transcript_path).resolve()) if current_transcript_path else "" + for path in transcript_paths: + scan = collector.collect_transcript_entries( + state_path=state_path, + sid=sid, + transcript_path=path, + max_lines=max_lines, + ) + entries = scan.get("entries", []) or [] + event_tool = tool_name if event_kind == "usage" and path == current_resolved else "model_request" + event_agent = active_agent if path == current_resolved else agent_identity.agent_for_transcript_path(path, active_agent) + + for idx, entry in enumerate(entries): + event_key = collector.build_transcript_event_key(kind=event_kind, tool=event_tool, entry=entry) + if not st.claim_transcript_event(state_path, sid, event_key, path): + continue + tokens = entry.get("tokens") + model = entry.get("model") + if event_kind == "stop": + event = emitter.build_stop_event( + sid=sid, + tokens=tokens, + model=model, + cost_usd=emitter.cost_of(tokens, model), + cost_session_usd=cost_session if idx == len(entries) - 1 else None, + active_agent=event_agent, + transcript_path=path, + source_offset=int(entry.get("offset", 0) or 0), + turn_id=turn_id, + message_id=str(entry.get("message_id") or "").strip() or None, + task_slug=task_slug, + stage=stage, + ) + else: + event = emitter.build_usage_event( + sid=sid, + tool=event_tool, + tokens=tokens, + active_agent=event_agent, + model=model, + cost_usd=emitter.cost_of(tokens, model), + transcript_path=path, + source_offset=int(entry.get("offset", 0) or 0), + turn_id=turn_id, + message_id=str(entry.get("message_id") or "").strip() or None, + task_slug=task_slug, + stage=stage, + ) + emitter.emit(log_path, event) + emitted_events.append(event) + + st.commit_transcript_progress( + state_path, + sid, + path, + offset=int(scan.get("new_offset", 0) or 0), + last_cumulative_usage=scan.get("last_cumulative"), + ) + return emitted_events + + +def main(argv: list[str]) -> int: + """读取 stdin 输入并按 phase 执行对应 hook 处理分支。""" + phase = normalize_phase(argv[0] if argv else "") + current_sid = "" + try: + data = collector.read_stdin_json() + current_sid = session_id_of(data) + if phase == "session-start": + handle_session_start(data) + elif phase == "user-prompt-submit": + handle_user_prompt_submit(data) + elif phase == "pre": + handle_pre(data) + elif phase == "post": + handle_post(data) + elif phase == "stop": + handle_stop(data) + else: + emitter.emit( + build_runtime_paths().log_path, + emitter.build_error_event(phase=phase or "unknown", error=f"Unsupported phase: {phase!r}", sid=current_sid), + ) + except Exception as exc: + emit_error(phase=phase or "unknown", sid=current_sid, err=exc) + return 0 diff --git a/.cursor/skills/agent-observability/scripts/core/scanner.py b/.cursor/skills/agent-observability/scripts/core/scanner.py new file mode 100644 index 0000000..86b5305 --- /dev/null +++ b/.cursor/skills/agent-observability/scripts/core/scanner.py @@ -0,0 +1,496 @@ +"""Skill / Rule 静态扫描层。 + +这一层只负责扫描项目里的 `.codebuddy/skills` 与 `.codebuddy/rules`： +- 识别有哪些可用 skill / rule +- 提取它们的基础元数据与 frontmatter +- 提供 path-based 命中推断依赖的 inventory 信息 + +它不依赖具体 workflow 语义，本身是通用层。 +""" +from __future__ import annotations + +import os +import re +from pathlib import Path +from typing import Any + +# --- 用于解析 SKILL.md / RULE.mdc frontmatter 的正则 --- +FRONTMATTER_RE = re.compile(r"^---\s*\n(.*?)\n---", re.S) +NAME_FIELD_RE = re.compile(r"^name:\s*(.+)$", re.M) +VERSION_FIELD_RE = re.compile(r"^version:\s*(.+)$", re.M) +AUTHOR_FIELD_RE = re.compile(r"^author:\s*(.+)$", re.M) +TAGS_FIELD_RE = re.compile(r"^tags:\s*\[([^\]]+)\]", re.M) +DESC_FIELD_RE = re.compile(r"^description:\s*(.+?)(?=\n\w+:|\n---|\Z)", re.S | re.M) +ALWAYS_APPLY_RE = re.compile(r"^alwaysApply:\s*(true|false)\s*$", re.M | re.I) +ENABLED_RE = re.compile(r"^enabled:\s*(true|false)\s*$", re.M | re.I) +GLOBS_RE = re.compile(r"^globs:\s*(.+?)(?=\n\w+:|\n---|\Z)", re.S | re.M) +APPLY_AGENTS_RE = re.compile(r"^applyAgents:\s*(.+?)(?=\n\w+:|\n---|\Z)", re.S | re.M) +EXCLUDE_AGENTS_RE = re.compile(r"^excludeAgents:\s*(.+?)(?=\n\w+:|\n---|\Z)", re.S | re.M) + +# 固定的 inventory 根目录（仅限项目内） +PROJECT_SKILLS_REL = Path(".codebuddy/skills") +PROJECT_RULES_REL = Path(".codebuddy/rules") + +# 支持识别的 skill 子模块根目录 +SUBMODULE_DIRS = {"references", "scripts", "templates", "examples", "checklists", "roles"} + +# 允许严格提取的路径类字段键名（不扫描自由文本） +PATH_VALUE_KEYS = { + "file_path", "filePath", "path", "target_file", "target_directory", + "cwd", "rule", "rule_path", "skill_path", +} +PATH_LIST_KEYS = {"paths", "files", "rules", "skills"} + +# 扫描时要跳过的目录 +SKIP_DIR_PARTS = {"skills-by-node", "node_modules", ".git", "__pycache__", "dist"} + +# 可能触发 skill 的工具名 +USE_SKILL_TOOLS = {"use_skill", "UseSkill", "load_skill", "Skill", "skill"} + + +def _is_skipped_path(p: Path) -> bool: + """判断路径是否命中扫描时应跳过的目录集合。""" + parts = set(p.parts) + return bool(parts & SKIP_DIR_PARTS) + + +def _parse_frontmatter(path: Path) -> dict[str, Any]: + """解析 `SKILL.md` 或 `RULE.mdc` 中的 frontmatter。""" + try: + text = path.read_text("utf-8", errors="ignore") + except Exception: + return {} + m = FRONTMATTER_RE.match(text) + if not m: + return {} + block = m.group(1) + info: dict = {} + + def _grab(regex, key, post=None): + mm = regex.search(block) + if mm: + v = mm.group(1).strip().strip('"\'').strip() + info[key] = post(v) if post else v + + _grab(NAME_FIELD_RE, "name") + _grab(VERSION_FIELD_RE, "version") + _grab(AUTHOR_FIELD_RE, "author") + _grab(TAGS_FIELD_RE, "tags", + lambda v: [t.strip().strip('"\'') for t in v.split(",") if t.strip()]) + _grab(DESC_FIELD_RE, "description", lambda v: v.strip().replace("\n", " ")[:200]) + return info + + +def _parse_list_field(block: str, regex: re.Pattern[str]) -> list[str]: + """从 frontmatter 文本块中解析一个逗号分隔的列表字段。""" + mm = regex.search(block) + if not mm: + return [] + raw = mm.group(1).strip().strip('"\'') + if raw.startswith("["): + raw = raw.strip("[]") + return [g.strip().strip('"\'').lower() for g in raw.split(",") if g.strip()] + + +def _parse_rule_frontmatter(rule_path: Path) -> dict[str, Any]: + """解析 `RULE.mdc` 中和规则生效相关的 frontmatter 字段。""" + try: + text = rule_path.read_text("utf-8", errors="ignore") + except Exception: + return {} + m = FRONTMATTER_RE.match(text) + if not m: + return {} + block = m.group(1) + info: dict = {} + + mm = ALWAYS_APPLY_RE.search(block) + if mm: + info["alwaysApply"] = mm.group(1).lower() == "true" + mm = ENABLED_RE.search(block) + if mm: + info["enabled"] = mm.group(1).lower() == "true" + globs = _parse_list_field(block, GLOBS_RE) + if globs: + info["globs"] = globs + + apply_agents = _parse_list_field(block, APPLY_AGENTS_RE) + if apply_agents: + info["applyAgents"] = apply_agents + + exclude_agents = _parse_list_field(block, EXCLUDE_AGENTS_RE) + if exclude_agents: + info["excludeAgents"] = exclude_agents + + mm = DESC_FIELD_RE.search(block) + if mm: + info["description"] = mm.group(1).strip().strip('"\'').replace("\n", " ")[:200] + return info + + +def _classify_source(path: Path, cwd: Path | None) -> str: + """识别来源类型；当前只支持项目内 `.codebuddy` 路径。""" + if not cwd: + return "unknown" + try: + path_str = str(path.resolve()) + cwd_str = str(cwd.resolve()) + except Exception: + return "unknown" + if path_str.startswith(cwd_str + "/.codebuddy/"): + return "project" + return "unknown" + + +def _collect_submodules(skill_dir: Path) -> dict[str, list]: + """收集一个 skill 内部的子模块目录，如 references/scripts/templates。""" + sub: dict = {} + if not skill_dir.is_dir(): + return sub + for kind in ("references", "scripts", "templates", "examples", "checklists", "roles"): + d = skill_dir / kind + if d.is_dir(): + files = [str(f.relative_to(skill_dir)) + for f in d.rglob("*") if f.is_file()] + if files: + sub[kind] = sorted(files)[:30] + return sub + + +def scan_skills_and_rules(cwd: str) -> tuple[dict[str, Any], dict[str, Any]]: + """扫描项目固定根目录：`.codebuddy/skills` 与 `.codebuddy/rules`。""" + skills: dict = {} + rules: dict = {} + cwd_path = Path(cwd) if cwd else None + if not cwd_path or not cwd_path.exists(): + return skills, rules + + skills_root = (cwd_path / PROJECT_SKILLS_REL).resolve() + rules_root = (cwd_path / PROJECT_RULES_REL).resolve() + + if skills_root.is_dir() and not _is_skipped_path(skills_root): + for p in skills_root.rglob("SKILL.md"): + if _is_skipped_path(p): + continue + skill_dir = p.parent + name = skill_dir.name + if not name or name in skills: + continue + meta = _parse_frontmatter(p) + skills[name] = { + "source": _classify_source(skill_dir, cwd_path), + "path": str(skill_dir), + "has_skill_md": True, + "version": meta.get("version"), + "author": meta.get("author"), + "description": meta.get("description"), + "submodules": _collect_submodules(skill_dir), + } + + if rules_root.is_dir() and not _is_skipped_path(rules_root): + for p in rules_root.rglob("*.mdc"): + if _is_skipped_path(p): + continue + name = p.parent.name if p.name == "RULE.mdc" else p.stem + if not name or name in rules: + continue + meta = _parse_rule_frontmatter(p) + rules[name] = { + "source": _classify_source(p.parent, cwd_path), + "path": str(p), + "alwaysApply": meta.get("alwaysApply", False), + "enabled": meta.get("enabled", True), + "globs": meta.get("globs") or [], + "applyAgents": meta.get("applyAgents") or [], + "excludeAgents": meta.get("excludeAgents") or [], + "description": meta.get("description"), + } + + return skills, rules + + +def lookup_skill_meta(skills_meta: dict[str, Any], name: str) -> dict[str, Any] | None: + """从缓存的 inventory 字典里按名称查找 skill 元数据。""" + meta = (skills_meta or {}).get(name) + return meta if isinstance(meta, dict) else None + + +def lookup_rule_meta(rules_meta: dict[str, Any], name: str) -> dict[str, Any] | None: + """从缓存的 inventory 字典里按名称查找 rule 元数据。""" + meta = (rules_meta or {}).get(name) + return meta if isinstance(meta, dict) else None + + +def _collect_path_strings(obj: Any) -> list[str]: + """只收集结构化路径字段，绝不解析自由文本。""" + out: list[str] = [] + + def _walk(x: Any) -> None: + if isinstance(x, dict): + for k, v in x.items(): + if k in PATH_VALUE_KEYS and isinstance(v, str) and v.strip(): + out.append(v.strip()) + elif k in PATH_LIST_KEYS: + if isinstance(v, str) and v.strip(): + out.append(v.strip()) + elif isinstance(v, list): + out.extend([ + s.strip() for s in v + if isinstance(s, str) and s.strip() + ]) + elif isinstance(v, (dict, list)): + _walk(v) + elif isinstance(x, list): + for it in x: + if isinstance(it, (dict, list)): + _walk(it) + + _walk(obj) + return list(dict.fromkeys(out)) + + +def _path_parts(path_str: str) -> list[str]: + """把路径拆成标准化片段列表，便于后续命中判断。""" + p = path_str.replace("\\", "/").strip() + return [part for part in p.split("/") if part] + + +def _extract_skill_from_path(path_str: str) -> str | None: + """从路径中抽取被引用的 skill 名。""" + parts = _path_parts(path_str) + for i in range(len(parts) - 2): + if parts[i] == ".codebuddy" and parts[i + 1] == "skills": + name = parts[i + 2] + if name and re.fullmatch(r"[a-zA-Z0-9_\-]+", name): + return name + return None + + +def _extract_rule_from_path(path_str: str) -> str | None: + """从路径中抽取被引用的 rule 名。""" + parts = _path_parts(path_str) + for i in range(len(parts) - 2): + if parts[i] == ".codebuddy" and parts[i + 1] == "rules": + name = parts[i + 2] + if name.lower().endswith(".mdc"): + name = Path(name).stem + if name and re.fullmatch(r"[a-zA-Z0-9_\-]+", name): + return name + return None + + +def _is_rule_allowed_for_agent(name: str, meta: dict[str, Any], active_agent: str | None) -> bool: + """按角色判断 rule 是否可用；`global` 永远保持可选。""" + if str(name).strip().lower() == "global": + return True + + agent = str(active_agent or "").strip().lower() + if not agent: + return True + + apply_agents = [str(a).strip().lower() for a in (meta.get("applyAgents") or []) if str(a).strip()] + exclude_agents = [str(a).strip().lower() for a in (meta.get("excludeAgents") or []) if str(a).strip()] + + if apply_agents and agent not in apply_agents: + return False + if exclude_agents and agent in exclude_agents: + return False + return True + + +def _glob_matches_any_path(globs: list[str], paths_to_check: list[str]) -> bool: + """判断配置的 glob 是否命中任一收集到的路径。""" + if not globs or not paths_to_check: + return False + import fnmatch + + for p in paths_to_check: + basename = os.path.basename(p) + if any(fnmatch.fnmatch(p, g) or fnmatch.fnmatch(basename, g) for g in globs): + return True + return False + + +def evaluate_rules_for_call( + data: dict, + rules_meta: dict, + active_agent: str | None = None, +) -> dict[str, dict[str, Any]]: + """评估当前工具调用下所有已知 rule，并保留未命中的原因。""" + if not rules_meta: + return {} + + tool_input = data.get("tool_input") or {} + paths_to_check = _collect_path_strings(tool_input) + _, path_rules = extract_paths_from_tool_call(data) + path_rule_set = set(path_rules) + + evaluations: dict[str, dict[str, Any]] = {} + for name, meta in (rules_meta or {}).items(): + if not isinstance(meta, dict): + continue + + globs = [str(g).strip() for g in (meta.get("globs") or []) if str(g).strip()] + apply_agents = [str(a).strip().lower() for a in (meta.get("applyAgents") or []) if str(a).strip()] + exclude_agents = [str(a).strip().lower() for a in (meta.get("excludeAgents") or []) if str(a).strip()] + evaluation: dict[str, Any] = {"matched": False} + + if name in path_rule_set: + evaluation["matched"] = True + evaluation["via"] = "path_inferred" + elif meta.get("enabled") is False: + evaluation["reason"] = "disabled" + elif not _is_rule_allowed_for_agent(name, meta, active_agent): + evaluation["reason"] = "agent_filtered" + if apply_agents: + evaluation["applyAgents"] = apply_agents + if exclude_agents: + evaluation["excludeAgents"] = exclude_agents + elif meta.get("alwaysApply"): + evaluation["matched"] = True + evaluation["via"] = "always_apply" + elif globs: + evaluation["globs"] = globs + evaluation["paths_checked"] = len(paths_to_check) + if not paths_to_check: + evaluation["reason"] = "no_paths" + elif _glob_matches_any_path(globs, paths_to_check): + evaluation["matched"] = True + evaluation["via"] = "glob" + else: + evaluation["reason"] = "glob_not_matched" + evaluation["paths_sample"] = paths_to_check[:3] + else: + evaluation["reason"] = "no_globs" + + evaluations[name] = evaluation + + return evaluations + + +def unmatched_rule_diagnostics_for_call( + data: dict, + rules_meta: dict, + active_agent: str | None = None, +) -> dict[str, dict[str, Any]]: + """返回当前工具调用下未命中的 rule 的精简诊断信息。""" + diagnostics: dict[str, dict[str, Any]] = {} + for name, evaluation in evaluate_rules_for_call(data, rules_meta, active_agent=active_agent).items(): + if evaluation.get("matched"): + continue + item: dict[str, Any] = {"reason": evaluation.get("reason") or "unknown"} + for key in ("globs", "paths_checked", "paths_sample", "applyAgents", "excludeAgents"): + value = evaluation.get(key) + if value is not None and value != []: + item[key] = value + diagnostics[name] = item + return diagnostics + + +def filtered_rules_for_agent(rules_meta: dict, active_agent: str | None = None) -> list[str]: + """列出因 agent 过滤而被排除的 rule（仅看已启用项）。""" + filtered: list[str] = [] + for name, evaluation in evaluate_rules_for_call({}, rules_meta, active_agent=active_agent).items(): + if evaluation.get("reason") == "agent_filtered": + filtered.append(name) + return sorted(set(filtered)) + + +def active_rules_for_call(data: dict, rules_meta: dict, active_agent: str | None = None) -> list[str]: + """判断给定工具调用下哪些 rule 处于生效状态。""" + active: list[str] = [] + for name, evaluation in evaluate_rules_for_call(data, rules_meta, active_agent=active_agent).items(): + if evaluation.get("matched") and evaluation.get("via") != "path_inferred": + active.append(name) + return sorted(set(active)) + + +def extract_skill_from_tool_call(data: dict) -> str | None: + """从类似 use_skill 的工具调用里提取 skill 名称。""" + tool = data.get("tool_name", "") + if tool not in USE_SKILL_TOOLS: + return None + tool_input = data.get("tool_input") or {} + if isinstance(tool_input, dict): + return tool_input.get("command") or tool_input.get("name") or tool_input.get("skill") + return None + + +def is_brainstorming_call(data: dict) -> bool: + """判断当前工具调用是否正在激活或使用 brainstorming skill。""" + direct = extract_skill_from_tool_call(data) + if isinstance(direct, str) and direct.strip().lower() == "brainstorming": + return True + + path_skills, _ = extract_paths_from_tool_call(data) + return any(str(s).strip().lower() == "brainstorming" for s in path_skills) + + +def extract_paths_from_tool_call(data: dict) -> tuple[list[str], list[str]]: + """只根据 `.codebuddy` 根目录下的结构化路径推断 skill/rule。""" + tool_input = data.get("tool_input") or {} + if not isinstance(tool_input, dict): + return [], [] + + skills: list[str] = [] + rules: list[str] = [] + for p in _collect_path_strings(tool_input): + s = _extract_skill_from_path(p) + if s: + skills.append(s) + r = _extract_rule_from_path(p) + if r: + rules.append(r) + + return list(dict.fromkeys(skills)), list(dict.fromkeys(rules)) + + +def extract_submodule_hits(data: dict) -> list[dict[str, str]]: + """仅从结构化 skill 路径中提取子模块命中结果，不扫描自由文本。""" + tool_input = data.get("tool_input") or {} + if not isinstance(tool_input, dict): + return [] + + seen: set[tuple[str, str]] = set() + hits: list[dict[str, str]] = [] + path_values = _collect_path_strings(tool_input) + + for p in path_values: + parts = _path_parts(p) + for i in range(len(parts) - 3): + if parts[i] == ".codebuddy" and parts[i + 1] == "skills": + skill_name = parts[i + 2] + sub_root = parts[i + 3] + if not re.fullmatch(r"[a-zA-Z0-9_\-]+", skill_name): + continue + if sub_root not in SUBMODULE_DIRS: + continue + sub_path = "/".join(parts[i + 3:]).rstrip("/") + key = (skill_name, sub_path) + if key in seen: + continue + seen.add(key) + hits.append({"skill": skill_name, "submodule": sub_path}) + break + + return hits + + +def extract_bash_skill_scripts(data: dict) -> list[str]: + """从命令文本中的显式 `.codebuddy/skills` 路径识别 skill 脚本。""" + if data.get("tool_name") not in ("Bash", "bash", "execute_command"): + return [] + tool_input = data.get("tool_input") or {} + if not isinstance(tool_input, dict): + return [] + cmd = tool_input.get("command") or "" + if not isinstance(cmd, str) or not cmd.strip(): + return [] + + skills: list[str] = [] + for token in cmd.split(): + t = token.strip("\"'`;,()[]{}") + s = _extract_skill_from_path(t) + if s: + skills.append(s) + return list(dict.fromkeys(skills)) diff --git a/.cursor/skills/agent-observability/scripts/core/state.py b/.cursor/skills/agent-observability/scripts/core/state.py new file mode 100644 index 0000000..5e7d33e --- /dev/null +++ b/.cursor/skills/agent-observability/scripts/core/state.py @@ -0,0 +1,1222 @@ +"""持久化状态层。 + +这一层负责跨 hook 调用保存与协调状态： +- session 级 skill/rule 使用统计 +- 当前 agent、派发关系与历史 +- transcript offset、去重 claim、pending emits +- AgentLens sidecar 状态 +""" +from __future__ import annotations + +import json +import time +from contextlib import contextmanager +from pathlib import Path +from typing import Any + +try: + import fcntl +except Exception: # pragma: no cover - non-posix fallback + fcntl = None # type: ignore + + +def get_state_path(base_dir: Path) -> Path: + """返回主状态文件路径。""" + log_dir = base_dir / "logs" + log_dir.mkdir(parents=True, exist_ok=True) + return log_dir / ".state.json" + + +def get_pending_path(base_dir: Path) -> Path: + """返回 Pre/PostToolUse 配对使用的 pending 文件路径。""" + log_dir = base_dir / "logs" + log_dir.mkdir(parents=True, exist_ok=True) + return log_dir / ".pending_calls.json" + + +def load_state(state_path: Path) -> dict[str, Any]: + """从磁盘加载状态；任何异常都回退为空字典。""" + if state_path.exists(): + try: + return json.loads(state_path.read_text("utf-8")) + except Exception: + return {} + return {} + + +def save_state(state_path: Path, state: dict[str, Any]) -> None: + """把状态写回磁盘。""" + state_path.write_text(json.dumps(state, ensure_ascii=False), encoding="utf-8") + + +def get_state_lock_path(state_path: Path) -> Path: + """返回状态写锁文件路径。""" + return state_path.parent / ".state.lock" + + +def get_transcript_seen_path(state_path: Path) -> Path: + """返回 transcript usage 事件 claim 的 sidecar 台账文件。 + + 这部分单独存放，不混进 `.state.json`，是为了避免并发 hook 写状态时， + 旧 state 覆盖掉已经 claim 过的 transcript offset。 + """ + return state_path.parent / ".transcript_events_seen.json" + + +@contextmanager +def state_lock(state_path: Path): + """为状态修改提供跨进程建议锁（独占）。""" + lock_path = get_state_lock_path(state_path) + lock_path.parent.mkdir(parents=True, exist_ok=True) + with lock_path.open("a+", encoding="utf-8") as fp: + if fcntl is not None: + fcntl.flock(fp.fileno(), fcntl.LOCK_EX) + try: + yield + finally: + if fcntl is not None: + fcntl.flock(fp.fileno(), fcntl.LOCK_UN) + + +@contextmanager +def state_read_lock(state_path: Path): + """为状态一致性读取提供跨进程共享读锁。""" + lock_path = get_state_lock_path(state_path) + lock_path.parent.mkdir(parents=True, exist_ok=True) + with lock_path.open("a+", encoding="utf-8") as fp: + if fcntl is not None: + fcntl.flock(fp.fileno(), fcntl.LOCK_SH) + try: + yield + finally: + if fcntl is not None: + fcntl.flock(fp.fileno(), fcntl.LOCK_UN) + + +def update_state_locked(state_path: Path, updater): + """在跨进程锁保护下完成 load -> mutate -> save。""" + with state_lock(state_path): + state = load_state(state_path) + result = updater(state) + save_state(state_path, state) + return result + + +def load_pending(pending_path: Path) -> dict[str, Any]: + """加载待配对的 pre/post 数据。""" + if pending_path.exists(): + try: + return json.loads(pending_path.read_text("utf-8")) + except Exception: + return {} + return {} + + +def save_pending(pending_path: Path, data: dict[str, Any]) -> None: + """保存待配对的 pre/post 数据。""" + pending_path.write_text(json.dumps(data), encoding="utf-8") + + +def ensure_session(state: dict, sid: str) -> dict: + """确保某个 session 的状态结构存在，并返回该 session 字典。 + + Session 结构： + { + "skills": {name: {count, first_ts, last_ts, tools, via, source, by_agent}}, + "rules": {name: {count, first_ts, last_ts, tools, via, source, by_agent}}, + "started_at": float, + "current_agent": str, + "agent_history": [{ts, agent, evidence}], + "dispatched": {agent_name: count}, + "_skills_meta": {...}, # SessionStart 时缓存的 inventory + "_rules_meta": {...}, + } + """ + if sid not in state: + state[sid] = { + "skills": {}, + "rules": {}, + "started_at": time.time(), + "current_agent": "main", + "agent_history": [], + "dispatched": {}, + } + sess = state[sid] + # 保证字段类型正确，兼容旧数据 + if not isinstance(sess.get("skills"), dict): + sess["skills"] = {} + if not isinstance(sess.get("rules"), dict): + sess["rules"] = {} + sess.setdefault("current_agent", "main") + sess.setdefault("agent_history", []) + sess.setdefault("dispatched", {}) + if not isinstance(sess.get("_pending_tool_emits"), list): + sess["_pending_tool_emits"] = [] + return sess + + +def get_devflow_state(state: dict[str, Any], sid: str) -> dict[str, Any]: + """返回某个 session 缓存的 devflow 上下文与最近一次 stage 快照（见 devflow.py）。""" + sess = state.get(sid) + if not isinstance(sess, dict): + return {} + dv = sess.get("_devflow") + return dv if isinstance(dv, dict) else {} + + +def set_devflow_state(state: dict[str, Any], sid: str, updates: dict[str, Any]) -> None: + """合并写入某个 session 的 devflow 缓存字段（`checked` / `context` / `last_snapshot`）。""" + sess = ensure_session(state, sid) + dv = sess.get("_devflow") + if not isinstance(dv, dict): + dv = {} + sess["_devflow"] = dv + dv.update(updates) + + +def get_agentlens_session(state: dict[str, Any], sid: str) -> dict[str, Any]: + """返回某个 session 的 AgentLens sidecar payload。""" + sess = ensure_session(state, sid) + payload = sess.setdefault("_agentlens", {}) + if not isinstance(payload, dict): + payload = {} + sess["_agentlens"] = payload + return payload + + +AGENTLENS_SCHEMA_VERSION = 2 +_TURN_HISTORY_MAX = 5 + + +def _migrate_agentlens_v1_to_v2(payload: dict[str, Any], sess: dict[str, Any] | None) -> dict[str, Any]: + """把 v1 版 ``_agentlens`` payload 转成以 turn 为中心的 v2 布局。 + + v1 布局（pre-turn）： + {"carrier": {...}, "enabled": bool, "session_id": "...", "app_name": "..."} + v2 布局（turn-centric）： + { + "version": 2, + "session_id": "...", + "enabled": bool, + "app_name": "...", + "current_turn": {"turn_id": "legacy", "started_at": ..., "carrier": {...}, "subagent_spans": {}, "agent_spans": {}, "step_spans": {}}, + "turn_history": [], + } + + 幂等约束：如果已经带有 ``version``，就原样返回。 + """ + if not isinstance(payload, dict): + return {"version": AGENTLENS_SCHEMA_VERSION, "enabled": False, "current_turn": None, "turn_history": []} + if payload.get("version") == AGENTLENS_SCHEMA_VERSION: + payload.setdefault("current_turn", None) + payload.setdefault("turn_history", []) + current = payload.get("current_turn") + if isinstance(current, dict): + current.setdefault("subagent_spans", {}) + current.setdefault("agent_spans", {}) + current.setdefault("step_spans", {}) + return payload + + migrated: dict[str, Any] = { + "version": AGENTLENS_SCHEMA_VERSION, + "session_id": payload.get("session_id"), + "enabled": bool(payload.get("enabled", False)), + } + if "app_name" in payload: + migrated["app_name"] = payload.get("app_name") + if "last_error" in payload: + migrated["last_error"] = payload.get("last_error") + + legacy_carrier = payload.get("carrier") + if isinstance(legacy_carrier, dict) and legacy_carrier: + started_at = None + if isinstance(sess, dict): + try: + started_at = float(sess.get("started_at") or 0) or None + except Exception: + started_at = None + migrated["current_turn"] = { + "turn_id": "legacy", + "started_at": started_at or time.time(), + "carrier": dict(legacy_carrier), + "subagent_spans": {}, + "agent_spans": {}, + "step_spans": {}, + } + else: + migrated["current_turn"] = None + migrated["turn_history"] = [] + return migrated + + +def load_agentlens_session(state_path: Path, sid: str) -> dict[str, Any]: + """加载某个 session 已持久化的 AgentLens sidecar payload。 + + 这里使用共享读锁，避免并发 ``update_agentlens_session`` 写入时读到撕裂数据。 + 返回值会透明地把 v1 payload 迁移成 v2 副本；真正落盘迁移会在下一次写入时完成。 + """ + with state_read_lock(state_path): + state = load_state(state_path) + sess = state.get(sid) + if not isinstance(sess, dict): + return {} + payload = sess.get("_agentlens") + if not isinstance(payload, dict): + return {} + # 先给调用方返回迁移后的副本，真正持久化迁移留到后续写入时完成。 + return _migrate_agentlens_v1_to_v2(dict(payload), sess) + + +def update_agentlens_session( + state_path: Path, + sid: str, + updates: dict[str, Any], + *, + clear_keys: list[str] | None = None, +) -> dict[str, Any]: + """原子地持久化某个 session 的 AgentLens sidecar 字段。 + + 在合并 updates 之前会先做 v1→v2 迁移，这样磁盘上的 payload 会随着写入自然收敛到 v2。 + """ + + def _update(state: dict[str, Any]) -> dict[str, Any]: + sess = ensure_session(state, sid) + payload = get_agentlens_session(state, sid) + # 如果仍是 v1，就原地迁移。 + if payload.get("version") != AGENTLENS_SCHEMA_VERSION: + migrated = _migrate_agentlens_v1_to_v2(dict(payload), sess) + payload.clear() + payload.update(migrated) + for key in clear_keys or []: + payload.pop(key, None) + for key, value in updates.items(): + if value is None: + payload.pop(key, None) + else: + payload[key] = value + payload["version"] = AGENTLENS_SCHEMA_VERSION + return dict(payload) + + return update_state_locked(state_path, _update) + + +def begin_turn( + state_path: Path, + sid: str, + turn_id: str, + carrier: dict[str, str], + *, + started_at: float | None = None, + max_history: int = _TURN_HISTORY_MAX, +) -> dict[str, Any]: + """在独占锁下打开一个新的 turn。 + + 语义： + - 如果存在 ``current_turn``，先把它归档进有上限的 ``turn_history``。 + - 为新 turn 重置 ``subagent_spans`` / ``agent_spans`` / ``step_spans``。 + - 对相同 ``turn_id`` 保持幂等：重复调用直接返回已有 ``current_turn``， + 不重新生成 trace_id。 + + 返回最终生成的 ``current_turn`` 字典。 + """ + ts = float(started_at if started_at is not None else time.time()) + + def _update(state: dict[str, Any]) -> dict[str, Any]: + sess = ensure_session(state, sid) + payload = get_agentlens_session(state, sid) + if payload.get("version") != AGENTLENS_SCHEMA_VERSION: + migrated = _migrate_agentlens_v1_to_v2(dict(payload), sess) + payload.clear() + payload.update(migrated) + + current = payload.get("current_turn") + # 幂等：如果同一个 turn_id 还在处理中，就原样返回。 + if isinstance(current, dict) and str(current.get("turn_id") or "") == str(turn_id): + return dict(current) + + # 归档上一个 turn。 + if isinstance(current, dict) and current.get("turn_id"): + history = payload.setdefault("turn_history", []) + if not isinstance(history, list): + history = [] + payload["turn_history"] = history + archived = { + "turn_id": current.get("turn_id"), + "started_at": current.get("started_at"), + "ended_at": ts, + "carrier_traceparent": (current.get("carrier") or {}).get("traceparent"), + } + history.append(archived) + if len(history) > max_history: + del history[: len(history) - max_history] + + new_turn = { + "turn_id": str(turn_id), + "started_at": ts, + "carrier": dict(carrier or {}), + "subagent_spans": {}, + "agent_spans": {}, + "step_spans": {}, + } + payload["current_turn"] = new_turn + payload["version"] = AGENTLENS_SCHEMA_VERSION + return dict(new_turn) + + return update_state_locked(state_path, _update) + + +def upsert_subagent_span( + state_path: Path, + sid: str, + role: str, + carrier: dict[str, str], +) -> dict[str, str]: + """在当前 turn 下登记 subagent 的 ``invoke_agent`` 子 span carrier。 + + 对 `(turn, role)` 维度保持幂等：如果活跃 turn 下已经记录了该角色的 carrier， + 就直接返回已有值，不再重复创建。 + """ + now = time.time() + + def _update(state: dict[str, Any]) -> dict[str, str]: + payload = get_agentlens_session(state, sid) + if payload.get("version") != AGENTLENS_SCHEMA_VERSION: + migrated = _migrate_agentlens_v1_to_v2(dict(payload), state.get(sid) if isinstance(state.get(sid), dict) else None) + payload.clear() + payload.update(migrated) + current = payload.get("current_turn") + if not isinstance(current, dict): + # 没有活跃 turn，无法登记 subagent span。 + return {} + subs = current.setdefault("subagent_spans", {}) + if not isinstance(subs, dict): + subs = {} + current["subagent_spans"] = subs + existing = subs.get(role) + if isinstance(existing, dict) and isinstance(existing.get("carrier"), dict) and existing["carrier"]: + return dict(existing["carrier"]) + subs[role] = {"carrier": dict(carrier or {}), "opened_at": now} + payload["version"] = AGENTLENS_SCHEMA_VERSION + return dict(carrier or {}) + + return update_state_locked(state_path, _update) + + +def upsert_subagent_span_atomic( + state_path: Path, + sid: str, + role: str, + *, + carrier_factory: Any, +) -> dict[str, str]: + """在写锁内原子地获取或创建 subagent 的 ``invoke_agent`` carrier。 + + `carrier_factory` 是一个无参可调用对象，用来生成 carrier 字典。 + 只有在 span 还不存在时才会被调用，从而避免这样的 TOCTOU 竞争： + ``_generate_subagent_carrier`` 先产生了不可逆的 zhiyanllm ``invoke_agent`` span， + 随后另一个 hook 进程已经写入同一角色，导致本次创建变成孤儿。 + """ + now = time.time() + + def _update(state: dict[str, Any]) -> dict[str, str]: + payload = get_agentlens_session(state, sid) + if payload.get("version") != AGENTLENS_SCHEMA_VERSION: + migrated = _migrate_agentlens_v1_to_v2(dict(payload), state.get(sid) if isinstance(state.get(sid), dict) else None) + payload.clear() + payload.update(migrated) + current = payload.get("current_turn") + if not isinstance(current, dict): + return {} + subs = current.setdefault("subagent_spans", {}) + if not isinstance(subs, dict): + subs = {} + current["subagent_spans"] = subs + existing = subs.get(role) + if isinstance(existing, dict) and isinstance(existing.get("carrier"), dict) and existing["carrier"]: + return dict(existing["carrier"]) + new_carrier = carrier_factory() + if not isinstance(new_carrier, dict) or not new_carrier.get("traceparent"): + return {} + subs[role] = {"carrier": dict(new_carrier), "opened_at": now} + payload["version"] = AGENTLENS_SCHEMA_VERSION + return dict(new_carrier) + + return update_state_locked(state_path, _update) + + +def get_current_turn(state_path: Path, sid: str) -> dict[str, Any] | None: + """返回当前活跃的 turn 字典；如果没有打开 turn，就返回 None。""" + payload = load_agentlens_session(state_path, sid) + current = payload.get("current_turn") + return current if isinstance(current, dict) else None + + +def get_subagent_span_carrier(state_path: Path, sid: str, role: str) -> dict[str, str] | None: + """返回活跃 turn 下已持久化的 subagent span carrier；如果没有则返回 None。""" + turn = get_current_turn(state_path, sid) + if not isinstance(turn, dict): + return None + subs = turn.get("subagent_spans") + if not isinstance(subs, dict): + return None + rec = subs.get(role) + if not isinstance(rec, dict): + return None + carrier = rec.get("carrier") + return dict(carrier) if isinstance(carrier, dict) and carrier else None + + +def _empty_agent_aggregate() -> dict[str, Any]: + """创建一份空的 agent 聚合指标骨架。""" + return { + "tokens": { + "input": 0, + "output": 0, + "cache_read": 0, + "cache_creation": 0, + "total": 0, + }, + "event_count": 0, + "cost_usd": 0.0, + "tool_duration_ms": 0, + "llm_call_count": 0, + "tool_call_count": 0, + "step_count": 0, + } + + +def _normalize_agent(agent: str | None) -> str: + """把 agent 名归一化，空值统一落到 ``main``。""" + return str(agent or "main").strip() or "main" + + +def _ensure_turn_agent_spans(current: dict[str, Any]) -> dict[str, Any]: + """确保当前 turn 下存在 `agent_spans` 容器。""" + agent_spans = current.setdefault("agent_spans", {}) + if not isinstance(agent_spans, dict): + agent_spans = {} + current["agent_spans"] = agent_spans + return agent_spans + + +def _ensure_agent_span_rec(current: dict[str, Any], agent: str) -> dict[str, Any]: + """确保指定 agent 在当前 turn 下有一条完整聚合记录。""" + agent_spans = _ensure_turn_agent_spans(current) + rec = agent_spans.get(agent) + if not isinstance(rec, dict): + rec = {} + agent_spans[agent] = rec + carrier = rec.get("carrier") + if not isinstance(carrier, dict): + rec["carrier"] = {} + aggregate = rec.get("aggregate") + if not isinstance(aggregate, dict): + rec["aggregate"] = _empty_agent_aggregate() + return rec + + +def get_agent_span_carrier(state_path: Path, sid: str, agent: str) -> dict[str, str] | None: + """返回活跃 turn 下已持久化的 agent 分组 span carrier；如果没有则返回 None。""" + turn = get_current_turn(state_path, sid) + if not isinstance(turn, dict): + return None + agent_spans = turn.get("agent_spans") + if not isinstance(agent_spans, dict): + return None + rec = agent_spans.get(_normalize_agent(agent)) + if not isinstance(rec, dict): + return None + carrier = rec.get("carrier") + return dict(carrier) if isinstance(carrier, dict) and carrier else None + + +def upsert_agent_span_atomic( + state_path: Path, + sid: str, + agent: str, + *, + carrier_factory: Any, +) -> dict[str, str]: + """原子地获取或创建 agent 分组 span carrier。""" + now = time.time() + normalized_agent = _normalize_agent(agent) + + def _update(state: dict[str, Any]) -> dict[str, str]: + payload = get_agentlens_session(state, sid) + if payload.get("version") != AGENTLENS_SCHEMA_VERSION: + migrated = _migrate_agentlens_v1_to_v2( + dict(payload), + state.get(sid) if isinstance(state.get(sid), dict) else None, + ) + payload.clear() + payload.update(migrated) + current = payload.get("current_turn") + if not isinstance(current, dict): + return {} + rec = _ensure_agent_span_rec(current, normalized_agent) + existing = rec.get("carrier") + if isinstance(existing, dict) and existing.get("traceparent"): + return dict(existing) + new_carrier = carrier_factory() + if not isinstance(new_carrier, dict) or not new_carrier.get("traceparent"): + return {} + rec["carrier"] = dict(new_carrier) + rec.setdefault("opened_at", now) + payload["version"] = AGENTLENS_SCHEMA_VERSION + return dict(new_carrier) + + return update_state_locked(state_path, _update) + + +def bump_agent_aggregate( + state_path: Path, + sid: str, + agent: str, + *, + usage_event: dict[str, Any] | None = None, + tool_event: dict[str, Any] | None = None, + step_created: bool = False, +) -> dict[str, Any]: + """增加活跃 turn 下按 agent 聚合的统计指标。""" + normalized_agent = _normalize_agent(agent) + + def _int(value: Any) -> int: + try: + return int(value or 0) + except Exception: + return 0 + + def _float(value: Any) -> float: + try: + return float(value or 0) + except Exception: + return 0.0 + + def _update(state: dict[str, Any]) -> dict[str, Any]: + payload = get_agentlens_session(state, sid) + if payload.get("version") != AGENTLENS_SCHEMA_VERSION: + migrated = _migrate_agentlens_v1_to_v2( + dict(payload), + state.get(sid) if isinstance(state.get(sid), dict) else None, + ) + payload.clear() + payload.update(migrated) + current = payload.get("current_turn") + if not isinstance(current, dict): + return {} + rec = _ensure_agent_span_rec(current, normalized_agent) + aggregate = rec.get("aggregate") + if not isinstance(aggregate, dict): + aggregate = _empty_agent_aggregate() + rec["aggregate"] = aggregate + tokens = aggregate.setdefault("tokens", {}) + if not isinstance(tokens, dict): + tokens = {} + aggregate["tokens"] = tokens + + if step_created: + aggregate["step_count"] = _int(aggregate.get("step_count")) + 1 + + if isinstance(tool_event, dict): + aggregate["tool_call_count"] = _int(aggregate.get("tool_call_count")) + 1 + ms = tool_event.get("ms") + if ms is not None: + aggregate["tool_duration_ms"] = _int(aggregate.get("tool_duration_ms")) + _int(ms) + + if isinstance(usage_event, dict): + raw_tokens = usage_event.get("tokens") + if isinstance(raw_tokens, dict): + # 聚合 agent 摘要里需要暴露的字段。 + for key in ("input", "output", "cache_read", "cache_creation"): + tokens[key] = _int(tokens.get(key)) + _int(raw_tokens.get(key)) + # total = input + output，其中 input 已经包含 cache_read。 + tokens["total"] = _int(tokens.get("input")) + _int(tokens.get("output")) + aggregate["llm_call_count"] = _int(aggregate.get("llm_call_count")) + 1 + aggregate["event_count"] = _int(aggregate.get("event_count")) + 1 + aggregate["cost_usd"] = round(_float(aggregate.get("cost_usd")) + _float(usage_event.get("cost_usd")), 10) + + payload["version"] = AGENTLENS_SCHEMA_VERSION + return json.loads(json.dumps(aggregate, ensure_ascii=False)) + + return update_state_locked(state_path, _update) + + +def get_subagent_aggregate(state_path: Path, sid: str, role: str) -> dict[str, Any] | None: + """返回活跃 turn 下当前 agent 的聚合摘要。""" + turn = get_current_turn(state_path, sid) + if not isinstance(turn, dict): + return None + agent_spans = turn.get("agent_spans") + if not isinstance(agent_spans, dict): + return None + rec = agent_spans.get(_normalize_agent(role)) + if not isinstance(rec, dict): + return None + aggregate = rec.get("aggregate") + if not isinstance(aggregate, dict): + return None + return json.loads(json.dumps(aggregate, ensure_ascii=False)) + + +def _step_span_key(agent: str, message_id: str) -> str: + """为 step span 生成按 agent 与 message_id 唯一定位的 key。""" + return f"{agent}|{message_id}" + + +def upsert_step_span( + state_path: Path, + sid: str, + agent: str, + message_id: str, + carrier: dict[str, str], + *, + transcript_path: str | None = None, +) -> dict[str, str]: + """在当前 turn 下登记一个 step span carrier，并对 `(turn, agent, message_id)` 保持幂等。""" + now = time.time() + normalized_agent = str(agent or "main").strip() or "main" + normalized_message_id = str(message_id or "").strip() + if not normalized_message_id: + return {} + + def _update(state: dict[str, Any]) -> dict[str, str]: + payload = get_agentlens_session(state, sid) + if payload.get("version") != AGENTLENS_SCHEMA_VERSION: + migrated = _migrate_agentlens_v1_to_v2(dict(payload), state.get(sid) if isinstance(state.get(sid), dict) else None) + payload.clear() + payload.update(migrated) + current = payload.get("current_turn") + if not isinstance(current, dict): + return {} + steps = current.setdefault("step_spans", {}) + if not isinstance(steps, dict): + steps = {} + current["step_spans"] = steps + key = _step_span_key(normalized_agent, normalized_message_id) + existing = steps.get(key) + if isinstance(existing, dict) and isinstance(existing.get("carrier"), dict) and existing["carrier"]: + return dict(existing["carrier"]) + steps[key] = { + "carrier": dict(carrier or {}), + "opened_at": now, + "agent": normalized_agent, + "message_id": normalized_message_id, + "transcript_path": transcript_path or "", + } + payload["version"] = AGENTLENS_SCHEMA_VERSION + return dict(carrier or {}) + + return update_state_locked(state_path, _update) + + +def upsert_step_span_atomic( + state_path: Path, + sid: str, + agent: str, + message_id: str, + *, + carrier_factory: Any, + transcript_path: str | None = None, +) -> dict[str, str]: + """在写锁内原子地获取或创建一个 step span carrier。 + + `carrier_factory` 是一个无参可调用对象，用来生成 carrier 字典。 + 只有在 span 还不存在时才会被调用，从而避免这样的 TOCTOU 竞争： + ``_generate_step_carrier`` 已经先产生了不可逆的 zhiyanllm span， + 但随后另一个 hook 进程已经写入同一 step，导致本次创建失去归属。 + """ + now = time.time() + normalized_agent = str(agent or "main").strip() or "main" + normalized_message_id = str(message_id or "").strip() + if not normalized_message_id: + return {} + + def _update(state: dict[str, Any]) -> dict[str, str]: + payload = get_agentlens_session(state, sid) + if payload.get("version") != AGENTLENS_SCHEMA_VERSION: + migrated = _migrate_agentlens_v1_to_v2(dict(payload), state.get(sid) if isinstance(state.get(sid), dict) else None) + payload.clear() + payload.update(migrated) + current = payload.get("current_turn") + if not isinstance(current, dict): + return {} + steps = current.setdefault("step_spans", {}) + if not isinstance(steps, dict): + steps = {} + current["step_spans"] = steps + key = _step_span_key(normalized_agent, normalized_message_id) + existing = steps.get(key) + if isinstance(existing, dict) and isinstance(existing.get("carrier"), dict) and existing["carrier"]: + return dict(existing["carrier"]) + new_carrier = carrier_factory() + if not isinstance(new_carrier, dict) or not new_carrier.get("traceparent"): + return {} + steps[key] = { + "carrier": dict(new_carrier), + "opened_at": now, + "agent": normalized_agent, + "message_id": normalized_message_id, + "transcript_path": transcript_path or "", + } + payload["version"] = AGENTLENS_SCHEMA_VERSION + return dict(new_carrier) + + return update_state_locked(state_path, _update) + + +def get_step_span_carrier(state_path: Path, sid: str, agent: str, message_id: str) -> dict[str, str] | None: + """返回活跃 turn 下已持久化的 step span carrier；如果没有则返回 None。""" + turn = get_current_turn(state_path, sid) + if not isinstance(turn, dict): + return None + steps = turn.get("step_spans") + if not isinstance(steps, dict): + return None + key = _step_span_key(str(agent or "main").strip() or "main", str(message_id or "").strip()) + rec = steps.get(key) + if not isinstance(rec, dict): + return None + carrier = rec.get("carrier") + return dict(carrier) if isinstance(carrier, dict) and carrier else None + + +def get_latest_session_id(state: dict[str, Any]) -> str | None: + """返回 state 中最近启动的 session id。""" + latest_sid: str | None = None + latest_started_at = -1.0 + for sid, sess in state.items(): + if sid.startswith("_") or not isinstance(sess, dict): + continue + started_at = float(sess.get("started_at", 0) or 0) + if started_at >= latest_started_at: + latest_started_at = started_at + latest_sid = sid + return latest_sid + + +def get_pending_tool_emits(state_path: Path, sid: str) -> list[dict[str, Any]]: + """返回等待与 transcript/message_id 关联的缓冲 tool 事件。""" + with state_read_lock(state_path): + state = load_state(state_path) + sess = state.get(sid) + if not isinstance(sess, dict): + return [] + pending = sess.get("_pending_tool_emits") + if not isinstance(pending, list): + return [] + return [dict(item) for item in pending if isinstance(item, dict)] + + +def append_pending_tool_emit(state_path: Path, sid: str, tool_event: dict[str, Any]) -> list[dict[str, Any]]: + """缓存一条 tool 事件，延后再做 message_id 关联。""" + + def _update(state: dict[str, Any]) -> list[dict[str, Any]]: + sess = ensure_session(state, sid) + pending = sess.setdefault("_pending_tool_emits", []) + if not isinstance(pending, list): + pending = [] + sess["_pending_tool_emits"] = pending + pending.append(dict(tool_event)) + return [dict(item) for item in pending if isinstance(item, dict)] + + return update_state_locked(state_path, _update) + + +def replace_pending_tool_emits(state_path: Path, sid: str, tool_events: list[dict[str, Any]]) -> list[dict[str, Any]]: + """覆盖某个 session 的缓冲 tool 事件队列。""" + + def _update(state: dict[str, Any]) -> list[dict[str, Any]]: + sess = ensure_session(state, sid) + normalized = [dict(item) for item in tool_events if isinstance(item, dict)] + sess["_pending_tool_emits"] = normalized + return [dict(item) for item in normalized] + + return update_state_locked(state_path, _update) + +def bump_skill( + sess: dict, + name: str, + *, + via: str, + tool: str | None, + meta: dict | None = None, + submodule: str | None = None, + agent: str | None = None, +) -> None: + """增加 skill 使用计数，并按需记录子模块与 agent 维度。""" + if not name: + return + now = time.time() + rec = sess["skills"].setdefault(name, { + "count": 0, + "first_ts": now, + "last_ts": now, + "tools": [], + "via": [], + "source": None, + "version": None, + "submodule_hits": {}, + "by_agent": {}, + }) + rec["count"] += 1 + rec["last_ts"] = now + if tool and tool not in rec["tools"]: + rec["tools"].append(tool) + if via not in rec.get("via", []): + rec.setdefault("via", []).append(via) + if meta: + if meta.get("source") and not rec.get("source"): + rec["source"] = meta["source"] + if meta.get("version") and not rec.get("version"): + rec["version"] = meta["version"] + # 子模块命中统计 + if submodule: + sub_dict = rec.setdefault("submodule_hits", {}) + if not isinstance(sub_dict, dict): + sub_dict = {} + rec["submodule_hits"] = sub_dict + sh = sub_dict.setdefault(submodule, { + "count": 0, "first_ts": now, "last_ts": now, "tools": [], + }) + sh["count"] += 1 + sh["last_ts"] = now + if tool and tool not in sh["tools"]: + sh["tools"].append(tool) + # 按 agent 维度统计 + if agent: + ag_dict = rec.setdefault("by_agent", {}) + if not isinstance(ag_dict, dict): + ag_dict = {} + rec["by_agent"] = ag_dict + ar = ag_dict.setdefault(agent, { + "count": 0, "first_ts": now, "last_ts": now, + "tools": [], "submodules": [], + }) + ar["count"] += 1 + ar["last_ts"] = now + if tool and tool not in ar["tools"]: + ar["tools"].append(tool) + if submodule and submodule not in ar["submodules"]: + ar["submodules"].append(submodule) + + +def bump_rule( + sess: dict, + name: str, + *, + via: str, + tool: str | None, + meta: dict | None = None, + agent: str | None = None, +) -> None: + """增加 rule 使用计数。""" + if not name: + return + now = time.time() + rec = sess["rules"].setdefault(name, { + "count": 0, + "first_ts": now, + "last_ts": now, + "tools": [], + "via": [], + "source": None, + "by_agent": {}, + }) + rec["count"] += 1 + rec["last_ts"] = now + if tool and tool not in rec["tools"]: + rec["tools"].append(tool) + if via not in rec.get("via", []): + rec.setdefault("via", []).append(via) + if meta and meta.get("source") and not rec.get("source"): + rec["source"] = meta["source"] + if agent: + ag = rec.setdefault("by_agent", {}) + if not isinstance(ag, dict): + ag = {} + rec["by_agent"] = ag + ar = ag.setdefault(agent, {"count": 0, "last_ts": now}) + ar["count"] += 1 + ar["last_ts"] = now + + +def prune_sessions(state: dict, max_sessions: int = 5) -> None: + """在 state 中只保留最近的 N 个 session。""" + # 先过滤掉非 session 键（以下划线开头） + session_keys = [k for k in state if not k.startswith("_")] + if len(session_keys) > max_sessions: + to_remove = sorted( + session_keys, + key=lambda s: state[s].get("started_at", 0) if isinstance(state[s], dict) else 0, + )[:len(session_keys) - max_sessions] + for k in to_remove: + del state[k] + + +def _transcript_scope_key(transcript_path: str | None = None) -> str: + """把 transcript 路径归一化成内部 scope key。""" + path = str(transcript_path or "").strip() + return path or "__session__" + + +def _ensure_transcript_scopes(sess: dict) -> dict[str, dict[str, Any]]: + """确保 session 下存在 transcript scope 容器，并兼容旧字段迁移。""" + scopes = sess.setdefault("_transcripts", {}) + if not isinstance(scopes, dict): + scopes = {} + sess["_transcripts"] = scopes + + legacy_scope = scopes.get("__session__") + if not isinstance(legacy_scope, dict): + legacy_scope = {} + scopes["__session__"] = legacy_scope + + if "offset" not in legacy_scope and "_transcript_offset" in sess: + legacy_scope["offset"] = int(sess.get("_transcript_offset", 0) or 0) + + legacy_usage = sess.get("_last_cumulative_usage") + if "last_cumulative_usage" not in legacy_scope and isinstance(legacy_usage, dict) and legacy_usage: + legacy_scope["last_cumulative_usage"] = legacy_usage + + legacy_dedup = sess.get("_last_usage_event") + if "last_usage_event" not in legacy_scope and isinstance(legacy_dedup, dict) and legacy_dedup: + legacy_scope["last_usage_event"] = legacy_dedup + + return scopes + + +def _get_transcript_scope( + sess: dict, + transcript_path: str | None = None, + *, + create: bool = False, +) -> dict[str, Any] | None: + """获取指定 transcript 的 scope；必要时按需创建。""" + scopes = _ensure_transcript_scopes(sess) + key = _transcript_scope_key(transcript_path) + scope = scopes.get(key) + if isinstance(scope, dict): + return scope + if create: + scope = {} + scopes[key] = scope + return scope + return None + + +def get_transcript_offset(state: dict, sid: str, transcript_path: str | None = None) -> int: + """获取某个 session/transcript 最近一次上报的 transcript 文件 offset。""" + sess = state.get(sid) + if not isinstance(sess, dict): + return 0 + + scope = _get_transcript_scope(sess, transcript_path) + if isinstance(scope, dict): + return int(scope.get("offset", 0) or 0) + + if _transcript_scope_key(transcript_path) == "__session__": + return int(sess.get("_transcript_offset", 0) or 0) + return 0 + + +def set_transcript_offset( + state: dict, + sid: str, + offset: int, + transcript_path: str | None = None, +) -> None: + """在成功上报后持久化 transcript 文件 offset。""" + sess = state.get(sid) + if not isinstance(sess, dict): + return + + scope = _get_transcript_scope(sess, transcript_path, create=True) + if isinstance(scope, dict): + scope["offset"] = int(offset or 0) + + if _transcript_scope_key(transcript_path) == "__session__": + sess["_transcript_offset"] = int(offset or 0) + + +def get_last_cumulative_usage( + state: dict, + sid: str, + transcript_path: str | None = None, +) -> dict[str, int]: + """获取某个 transcript 最近一次上报的累计 token usage。""" + sess = state.get(sid) + if not isinstance(sess, dict): + return {} + + scope = _get_transcript_scope(sess, transcript_path) + if isinstance(scope, dict): + v = scope.get("last_cumulative_usage") + if isinstance(v, dict): + return v + + if _transcript_scope_key(transcript_path) == "__session__": + v = sess.get("_last_cumulative_usage") + if isinstance(v, dict): + return v + return {} + + +def get_last_llm_offset( + state: dict, + sid: str, + transcript_path: str | None = None, +) -> int: + """获取最近一次处理到的 LLM source_offset，用于增量构建 I/O。""" + sess = state.get(sid) + if not isinstance(sess, dict): + return 0 + scope = _get_transcript_scope(sess, transcript_path) + if isinstance(scope, dict): + v = scope.get("last_llm_offset") + if isinstance(v, (int, float)): + return int(v) + return 0 + + +def set_last_llm_offset( + state: dict, + sid: str, + offset: int, + transcript_path: str | None = None, +) -> None: + """持久化某个 transcript 最近处理到的 LLM source_offset。""" + sess = state.get(sid) + if not isinstance(sess, dict): + return + scope = _get_transcript_scope(sess, transcript_path, create=True) + if isinstance(scope, dict): + scope["last_llm_offset"] = int(offset or 0) + + +def set_last_cumulative_usage( + state: dict, + sid: str, + usage: dict[str, int], + transcript_path: str | None = None, +) -> None: + """在处理完一个 turn 后持久化累计 token usage。""" + sess = state.get(sid) + if not isinstance(sess, dict): + return + + scope = _get_transcript_scope(sess, transcript_path, create=True) + if isinstance(scope, dict): + scope["last_cumulative_usage"] = usage + + if _transcript_scope_key(transcript_path) == "__session__": + sess["_last_cumulative_usage"] = usage + + +def get_last_usage_event( + state: dict, + sid: str, + transcript_path: str | None = None, +) -> dict[str, Any] | None: + """获取某个 transcript 最近一次用于去重的 usage 快照。""" + sess = state.get(sid) + if not isinstance(sess, dict): + return None + + scope = _get_transcript_scope(sess, transcript_path) + if isinstance(scope, dict): + event = scope.get("last_usage_event") + if isinstance(event, dict): + return event + + if _transcript_scope_key(transcript_path) == "__session__": + event = sess.get("_last_usage_event") + if isinstance(event, dict): + return event + return None + + +def set_last_usage_event( + state: dict, + sid: str, + usage_event: dict[str, Any], + transcript_path: str | None = None, +) -> None: + """持久化某个 transcript 最近一次用于去重的 usage 快照。""" + sess = state.get(sid) + if not isinstance(sess, dict): + return + + scope = _get_transcript_scope(sess, transcript_path, create=True) + if isinstance(scope, dict): + scope["last_usage_event"] = usage_event + + if _transcript_scope_key(transcript_path) == "__session__": + sess["_last_usage_event"] = usage_event + + +def claim_transcript_event( + state_path: Path, + sid: str, + event_key: str, + transcript_path: str | None = None, + *, + max_seen_keys: int = 512, +) -> bool: + """跨进程原子地声明一个 transcript 作用域内的事件键。""" + seen_path = get_transcript_seen_path(state_path) + scope_key = f"{sid}|{_transcript_scope_key(transcript_path)}" + + with state_lock(state_path): + if seen_path.exists(): + try: + ledger = json.loads(seen_path.read_text("utf-8")) + except Exception: + ledger = {} + else: + ledger = {} + if not isinstance(ledger, dict): + ledger = {} + + seen = ledger.setdefault(scope_key, {}) + if not isinstance(seen, dict): + seen = {} + ledger[scope_key] = seen + if event_key in seen: + return False + seen[event_key] = time.time() + if len(seen) > max_seen_keys: + stale_keys = sorted(seen, key=lambda k: float(seen.get(k, 0) or 0))[:-max_seen_keys] + for key in stale_keys: + seen.pop(key, None) + seen_path.write_text(json.dumps(ledger, ensure_ascii=False), encoding="utf-8") + return True + + +def commit_transcript_progress( + state_path: Path, + sid: str, + transcript_path: str | None = None, + *, + offset: int | None = None, + last_cumulative_usage: dict[str, int] | None = None, +) -> None: + """在处理完一个窗口后，原子地持久化 transcript 解析进度。""" + + def _update(state: dict[str, Any]) -> None: + sess = ensure_session(state, sid) + scope = _get_transcript_scope(sess, transcript_path, create=True) + if not isinstance(scope, dict): + return + if offset is not None: + current_offset = int(scope.get("offset", 0) or 0) + next_offset = int(offset or 0) + if next_offset > current_offset: + scope["offset"] = next_offset + if _transcript_scope_key(transcript_path) == "__session__": + sess["_transcript_offset"] = next_offset + if isinstance(last_cumulative_usage, dict) and last_cumulative_usage: + scope["last_cumulative_usage"] = last_cumulative_usage + if _transcript_scope_key(transcript_path) == "__session__": + sess["_last_cumulative_usage"] = last_cumulative_usage + + update_state_locked(state_path, _update) diff --git a/.cursor/skills/agent-observability/scripts/dashboard/index.html b/.cursor/skills/agent-observability/scripts/dashboard/index.html new file mode 100644 index 0000000..8c391f5 --- /dev/null +++ b/.cursor/skills/agent-observability/scripts/dashboard/index.html @@ -0,0 +1,1086 @@ + + + + + +Agent Observability + + + + + +
+
+
+
Agent Observability
+
读取本地 dashboard-data.json（由 build_dashboard_data.py 从 metrics.ndjson / workflow-state.json 聚合而成）· LoopForge 项目
+
+
加载中…
+
+ +
+ 时间范围 + + + +
+ +
+ +
+ + + + + +
+ +
+
+
建议关注按当前范围内的阈值规则实时算出，不是固定文案
+
+
+ +
+
+ 每日成本 +
cost_usd（按 config/pricing.json 估算）
+
+
+
+ +
+
+
+ Token 构成 +
+
+
+
+
+
按模型的成本占比Top 4 + 其他
+
+
+
+ +
+
数据表图表的完整数值备份
+
日期会话数成本 (USD)inputoutputcache_read工具失败
+
+
+ + + + + + + + + +
+ 数据来源：python3 ../build_dashboard_data.py --project-root <devflow 项目根目录> 会读取该项目下 + .codebuddy/skills/agent-observability/logs/metrics.ndjson、.state.json、 + artifacts/*/workflow-state.json（Classic/Portable 两种 schema 自动识别）与 + .codebuddy/hooks/logs/auto-dispatch.log，聚合成同目录下的 dashboard-data.json。 + 这个页面只读那份快照，不直接碰任何日志/状态文件——重新生成快照后刷新页面即可看到最新数据。 + 全部处理在本机完成，不发往任何外部服务。 +
+
+ + + + diff --git a/.cursor/skills/agent-observability/scripts/main.py b/.cursor/skills/agent-observability/scripts/main.py new file mode 100644 index 0000000..4ca2481 --- /dev/null +++ b/.cursor/skills/agent-observability/scripts/main.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +"""agent-observability 的 hook 入口。""" +from __future__ import annotations + +import sys +from pathlib import Path + + +if __package__ in (None, ""): + _SCRIPTS_DIR = Path(__file__).resolve().parent + if str(_SCRIPTS_DIR) not in sys.path: + sys.path.insert(0, str(_SCRIPTS_DIR)) + from core.runtime import main as runtime_main # type: ignore +else: + from .core.runtime import main as runtime_main + + +def main() -> int: + return runtime_main(sys.argv[1:]) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.cursor/skills/agent-observability/scripts/run_hook.sh b/.cursor/skills/agent-observability/scripts/run_hook.sh new file mode 100644 index 0000000..d476810 --- /dev/null +++ b/.cursor/skills/agent-observability/scripts/run_hook.sh @@ -0,0 +1,16 @@ +#!/bin/sh +set -eu + +PROJECT_DIR="${CODEBUDDY_PROJECT_DIR:-$(cd "$(dirname "$0")"/../../../.. && pwd -P)}" +PYTHON_BIN="$PROJECT_DIR/.venv/bin/python" +MAIN_PY="$PROJECT_DIR/.codebuddy/skills/agent-observability/scripts/main.py" + +# 只有当 .venv 里可用 zhiyanllm 时才优先使用该解释器； +# 否则回退到已安装 zhiyanllm 的系统 python3。 +if [ -x "$PYTHON_BIN" ]; then + if "$PYTHON_BIN" -c "import zhiyanllm" 2>/dev/null; then + exec "$PYTHON_BIN" "$MAIN_PY" "$@" + fi +fi + +exec python3 "$MAIN_PY" "$@" diff --git a/.cursor/skills/agent-observability/templates/settings-hook.json b/.cursor/skills/agent-observability/templates/settings-hook.json new file mode 100644 index 0000000..a76fc26 --- /dev/null +++ b/.cursor/skills/agent-observability/templates/settings-hook.json @@ -0,0 +1,62 @@ +{ + "_comment": "将本文件合并到 ~/.cursor/settings.json 或 /.cursor/settings.json", + "hooks": { + "SessionStart": [ + { + "hooks": [ + { + "type": "command", + "command": "\"$CODEBUDDY_PROJECT_DIR\"/.cursor/skills/agent-observability/scripts/run_hook.sh session-start", + "timeout": 5 + } + ] + } + ], + "UserPromptSubmit": [ + { + "hooks": [ + { + "type": "command", + "command": "\"$CODEBUDDY_PROJECT_DIR\"/.cursor/skills/agent-observability/scripts/run_hook.sh user-prompt-submit", + "timeout": 5 + } + ] + } + ], + "PreToolUse": [ + { + "matcher": "", + "hooks": [ + { + "type": "command", + "command": "\"$CODEBUDDY_PROJECT_DIR\"/.cursor/skills/agent-observability/scripts/run_hook.sh pre", + "timeout": 5 + } + ] + } + ], + "PostToolUse": [ + { + "matcher": "", + "hooks": [ + { + "type": "command", + "command": "\"$CODEBUDDY_PROJECT_DIR\"/.cursor/skills/agent-observability/scripts/run_hook.sh post", + "timeout": 30 + } + ] + } + ], + "Stop": [ + { + "hooks": [ + { + "type": "command", + "command": "\"$CODEBUDDY_PROJECT_DIR\"/.cursor/skills/agent-observability/scripts/run_hook.sh stop", + "timeout": 30 + } + ] + } + ] + } +} diff --git a/.cursor/skills/agent-observability/tests/fixtures/transcript.jsonl b/.cursor/skills/agent-observability/tests/fixtures/transcript.jsonl new file mode 100644 index 0000000..a5ea47d --- /dev/null +++ b/.cursor/skills/agent-observability/tests/fixtures/transcript.jsonl @@ -0,0 +1,10 @@ +{"type":"system","subtype":"init","session_id":"demo-001","content":"\ndo not exec\n\n\npdf\nhermes-devflow\n"} +{"type":"assistant","providerData":{"messageId":"msg-001"},"content":[{"type":"tool_use","name":"read_file","input":{"filePath":"README.md"}}],"usage":{"input_tokens":1820,"output_tokens":42,"cache_read_input_tokens":15300}} +{"type":"function_call","name":"Read","callId":"call-001","providerData":{"messageId":"msg-001"}} +{"type":"function_call_result","name":"Read","callId":"call-001","providerData":{"messageId":"msg-001"}} +{"type":"assistant","providerData":{"messageId":"msg-002"},"content":[{"type":"tool_use","name":"use_skill","input":{"command":"pdf"}}],"usage":{"input_tokens":2100,"output_tokens":58,"cache_read_input_tokens":15300}} +{"type":"function_call","name":"Bash","callId":"call-002","providerData":{"messageId":"msg-002"}} +{"type":"function_call","name":"Bash","callId":"call-003","providerData":{"messageId":"msg-002"}} +{"type":"function_call_result","name":"Bash","callId":"call-002","providerData":{"messageId":"msg-002"}} +{"type":"function_call_result","name":"Bash","callId":"call-003","providerData":{"messageId":"msg-002"}} +{"type":"assistant","providerData":{"messageId":"msg-003"},"content":"完成","usage":{"input_tokens":2240,"output_tokens":120,"cache_read_input_tokens":15300}} diff --git a/.cursor/skills/agent-observability/tests/test_agent_identity.py b/.cursor/skills/agent-observability/tests/test_agent_identity.py new file mode 100644 index 0000000..664cc1d --- /dev/null +++ b/.cursor/skills/agent-observability/tests/test_agent_identity.py @@ -0,0 +1,84 @@ +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path +import sys + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "scripts" +if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) + +from core.agent_identity import AgentIdentityResolver # type: ignore +from core import state as st # type: ignore + + +class AgentIdentityResolverTests(unittest.TestCase): + def test_extract_identity_from_send_message_json(self): + resolver = AgentIdentityResolver() + data = { + "tool_name": "send_message", + "tool_input": { + "content": '{"from_role":"developer","next_target":{"role_name":"leader"}}' + }, + } + current, dispatched = resolver.extract_identity(data) + self.assertEqual(current, "developer") + self.assertEqual(dispatched, "leader") + + def test_track_returns_active_and_dispatched(self): + resolver = AgentIdentityResolver() + with tempfile.TemporaryDirectory() as td: + state_path = Path(td) / ".state.json" + sid = "s2" + state_data = st.load_state(state_path) + st.ensure_session(state_data, sid) + st.save_state(state_path, state_data) + + data = { + "tool_name": "Task", + "tool_input": {"subagent_name": "developer"}, + } + active, dispatched = resolver.track(state_path=state_path, sid=sid, data=data) + self.assertEqual(active, "main") + self.assertEqual(dispatched, "developer") + + state2 = st.load_state(state_path) + sess = state2.get(sid, {}) + self.assertEqual(sess.get("current_agent"), "developer") + self.assertEqual(sess.get("dispatched", {}).get("developer"), 1) + + def test_track_recognizes_agent_tool_subagent_type(self): + """Claude Code 的 Agent 工具用 subagent_type（不是 CodeBuddy 原生 Task 的 + subagent_name）标识派发目标；这条路径之前没有测试覆盖，_extract_from_task + 漏认这个字段会导致 dispatch 统计对所有 Agent 工具派发的场景失明。""" + resolver = AgentIdentityResolver() + with tempfile.TemporaryDirectory() as td: + state_path = Path(td) / ".state.json" + sid = "s3" + state_data = st.load_state(state_path) + st.ensure_session(state_data, sid) + st.save_state(state_path, state_data) + + data = { + "tool_name": "Agent", + "tool_input": {"subagent_type": "worker", "description": "spawn a team"}, + } + active, dispatched = resolver.track(state_path=state_path, sid=sid, data=data) + self.assertEqual(active, "main") + self.assertEqual(dispatched, "worker") + + state2 = st.load_state(state_path) + sess = state2.get(sid, {}) + self.assertEqual(sess.get("dispatched", {}).get("worker"), 1) + hist = sess.get("agent_history") or [] + self.assertTrue(any( + h.get("agent") == "worker" and str(h.get("evidence") or "").startswith("dispatch") + for h in hist + )) + + def test_extract_from_task_prefers_subagent_type_over_subagent_name(self): + resolver = AgentIdentityResolver() + result = resolver._extract_from_task({"subagent_type": "explorer", "subagent_name": "developer"}) + self.assertEqual(result, "explorer") diff --git a/.cursor/skills/agent-observability/tests/test_build_dashboard_data.py b/.cursor/skills/agent-observability/tests/test_build_dashboard_data.py new file mode 100644 index 0000000..53b031c --- /dev/null +++ b/.cursor/skills/agent-observability/tests/test_build_dashboard_data.py @@ -0,0 +1,603 @@ +from __future__ import annotations + +import contextlib +import io +import json +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "scripts" +if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) + +import build_dashboard_data as bdd # type: ignore + + +class ToolCallFailedTests(unittest.TestCase): + """真实 CodeBuddy CLI transcript 里从没出现过 `is_error` 这个键——实测抓到的 + rawResponse 形如 {"exitCode": 0, "tool_error_code": "0", ...}（成功）或非 0 + exitCode（失败）。之前只认 is_error，导致这个宿主上失败统计永远是 0。""" + + def test_real_success_shape_from_codebuddy_is_not_a_failure(self): + # 实测数据：ls 一个不存在的路径，但命令写成了 `ls ...; echo "EXIT_CODE=$?"`， + # 复合命令整体 exitCode 被内层 echo 冲成了 0——工具调用本身没有失败。 + raw_response = { + "exitCode": 0, "signal": None, "interrupted": False, + "sandboxDenied": False, "stderrBytesTruncated": 0, + "stdoutBytesTruncated": 0, "tool_error_code": "0", + } + self.assertFalse(bdd.tool_call_failed(raw_response)) + + def test_nonzero_exit_code_is_a_failure(self): + raw_response = {"exitCode": 1, "tool_error_code": "1"} + self.assertTrue(bdd.tool_call_failed(raw_response)) + + def test_nonzero_tool_error_code_without_exit_code_is_a_failure(self): + raw_response = {"tool_error_code": "127"} + self.assertTrue(bdd.tool_call_failed(raw_response)) + + def test_legacy_is_error_flag_still_recognized(self): + self.assertTrue(bdd.tool_call_failed({"is_error": True})) + + def test_missing_or_non_dict_raw_response_is_not_a_failure(self): + self.assertFalse(bdd.tool_call_failed(None)) + self.assertFalse(bdd.tool_call_failed("not a dict")) + self.assertFalse(bdd.tool_call_failed({})) + + def test_build_failures_and_session_status_use_exit_code(self): + events = [ + { + "event": "tool", "sid": "s1", "ts": 1.0, "tool": "Bash", + "tool_details": {"raw_response": {"exitCode": 1, "tool_error_code": "1"}}, + }, + ] + failures = bdd.build_failures(events) + self.assertEqual(len(failures), 1) + self.assertEqual(failures[0]["tool"], "Bash") + self.assertEqual(failures[0]["code"], "1") + + _, sessions = bdd.build_daily_and_sessions(events) + self.assertEqual(sessions[0]["status"], "error") + + +class ResolveInputPathsTests(unittest.TestCase): + def setUp(self): + self.skill_root = Path("/tmp/fake-skill-root") + + def test_defaults_fall_back_to_skill_root_logs(self): + metrics, state = bdd.resolve_input_paths(self.skill_root) + self.assertEqual(metrics, self.skill_root / "logs" / "metrics.ndjson") + self.assertEqual(state, self.skill_root / "logs" / ".state.json") + + def test_none_args_equivalent_to_empty_args(self): + m1, s1 = bdd.resolve_input_paths(self.skill_root, None, None) + m2, s2 = bdd.resolve_input_paths(self.skill_root, "", "") + self.assertEqual((m1, s1), (m2, s2)) + + def test_both_args_override(self): + metrics, state = bdd.resolve_input_paths( + self.skill_root, "/abs/a/metrics.ndjson", "/abs/b/.state.json" + ) + self.assertEqual(metrics, Path("/abs/a/metrics.ndjson").absolute()) + self.assertEqual(state, Path("/abs/b/.state.json").absolute()) + + def test_metrics_only_override_keeps_default_state(self): + metrics, state = bdd.resolve_input_paths( + self.skill_root, "/abs/a/metrics.ndjson", None + ) + self.assertEqual(metrics, Path("/abs/a/metrics.ndjson").absolute()) + self.assertEqual(state, self.skill_root / "logs" / ".state.json") + + def test_state_only_override_keeps_default_metrics(self): + metrics, state = bdd.resolve_input_paths( + self.skill_root, None, "/abs/b/.state.json" + ) + self.assertEqual(metrics, self.skill_root / "logs" / "metrics.ndjson") + self.assertEqual(state, Path("/abs/b/.state.json").absolute()) + + def test_tilde_expansion(self): + metrics, state = bdd.resolve_input_paths( + self.skill_root, "~/x/metrics.ndjson", "~/y/.state.json" + ) + self.assertEqual(metrics, (Path.home() / "x" / "metrics.ndjson").absolute()) + self.assertEqual(state, (Path.home() / "y" / ".state.json").absolute()) + + +class MainEndToEndTests(unittest.TestCase): + def _run_main(self, argv: list[str]) -> None: + old_argv = sys.argv + sys.argv = ["build_dashboard_data.py"] + argv + try: + self.assertEqual(bdd.main(), 0) + finally: + sys.argv = old_argv + + def test_default_paths_unchanged_and_reflected_in_source(self): + with tempfile.TemporaryDirectory() as td: + out = Path(td) / "dashboard-data.json" + self._run_main(["--project-root", td, "--out", str(out)]) + self.assertTrue(out.is_file()) + data = json.loads(out.read_text("utf-8")) + expected_metrics = str(bdd.SCRIPTS_DIR.parent / "logs" / "metrics.ndjson") + expected_state = str(bdd.SCRIPTS_DIR.parent / "logs" / ".state.json") + self.assertEqual(data["source"]["metrics_ndjson"], expected_metrics) + self.assertEqual(data["source"]["state_json"], expected_state) + + def test_overridden_paths_read_and_reflected_in_source(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + metrics_file = root / "metrics.ndjson" + state_file = root / ".state.json" + metrics_file.write_text( + '{"event":"usage","sid":"s1","ts":1,"tokens":{"input":10,"output":20}}\n', + encoding="utf-8", + ) + state_file.write_text(json.dumps({"s1": {"skills": {}}}), encoding="utf-8") + out = root / "dashboard-data.json" + + self._run_main([ + "--project-root", td, + "--out", str(out), + "--metrics-path", str(metrics_file), + "--state-path", str(state_file), + ]) + self.assertTrue(out.is_file()) + + data = json.loads(out.read_text("utf-8")) + # resolve_input_paths 对覆盖路径执行 .expanduser().absolute()，断言需对齐（macOS 下不用 .resolve() 以避免 /tmp → /private/tmp 符号链接错位）。 + self.assertEqual(data["source"]["metrics_ndjson"], str(metrics_file.absolute())) + self.assertEqual(data["source"]["state_json"], str(state_file.absolute())) + # 覆盖文件确实被读取：metrics 行被解析进了 event_count。 + self.assertEqual(data["source"]["event_count"], 1) + + +class TopSlowTests(unittest.TestCase): + """`--top-slow N` 是纯增量能力：默认不开启，开启后也只往终端打印，绝不改动 + dashboard-data.json 的结构（看板前端按字段取值，多一个字段或少一个字段都会 + 直接影响渲染）。这些用例把"降序取前 N""非 tool 事件排除""ms 兜底""只读" + 四条边界钉住，避免后续有人顺手把结果写进 JSON 或改了排序方向。""" + + def _events(self): + return [ + {"event": "user_prompt_submit", "sid": "s1", "ts": 1.0, "turn_id": "t1", "agent": "main"}, + {"event": "tool", "sid": "s1", "ts": 2.0, "tool": "Read", "ms": 300, "agent": "main"}, + {"event": "usage", "sid": "s1", "ts": 3.0, "tokens": {"input": 10, "output": 5}, "agent": "main"}, + {"event": "tool", "sid": "s1", "ts": 4.0, "tool": "Bash", "ms": 900, "agent": "main"}, + {"event": "tool", "sid": "s1", "ts": 5.0, "tool": "Grep", "ms": 120, "agent": "main"}, + ] + + def _write_metrics(self, root: Path) -> Path: + metrics = root / "metrics.ndjson" + metrics.write_text( + "\n".join(json.dumps(e) for e in self._events()) + "\n", + encoding="utf-8", + ) + return metrics + + def _run_main(self, argv: list[str]) -> str: + """跑一次 main() 并返回捕获到的 stdout——top-slow 的产出只体现在终端上， + 不体现在 JSON 里，所以断言必须落在 stdout。""" + old_argv = sys.argv + sys.argv = ["build_dashboard_data.py"] + argv + buf = io.StringIO() + try: + with contextlib.redirect_stdout(buf): + rc = bdd.main() + finally: + sys.argv = old_argv + self.assertEqual(rc, 0) + return buf.getvalue() + + def _printed_tools(self, stdout: str) -> list[str]: + tools = [] + for line in stdout.splitlines(): + line = line.strip() + if not line or not line[0].isdigit(): + continue + tools.append(line.split(". ", 1)[1].split()[0]) + return tools + + def test_returns_n_slowest_tool_events_descending(self): + rows = bdd.build_top_slow(self._events(), 2) + self.assertEqual([(r["tool"], r["ms"]) for r in rows], [("Bash", 900), ("Read", 300)]) + + def test_limit_greater_than_event_count_returns_all_without_error(self): + events = [{"event": "tool", "sid": "s1", "ts": 1.0, "tool": "Bash", "ms": 10}] + rows = bdd.build_top_slow(events, 99) + self.assertEqual(len(rows), 1) + self.assertEqual(rows[0]["tool"], "Bash") + + def test_non_tool_events_are_excluded(self): + rows = bdd.build_top_slow(self._events(), 10) + self.assertEqual([r["tool"] for r in rows], ["Bash", "Read", "Grep"]) + # usage 事件即使带 ms 也不能混进来。 + self.assertTrue(all(isinstance(r["ms"], (int, float)) for r in rows)) + + def test_missing_or_non_numeric_ms_falls_back_to_zero(self): + events = [ + {"event": "tool", "sid": "s1", "ts": 1.0, "tool": "Bash"}, + {"event": "tool", "sid": "s1", "ts": 2.0, "tool": "Read", "ms": "120"}, + {"event": "tool", "sid": "s1", "ts": 3.0, "tool": "Grep", "ms": None}, + {"event": "tool", "sid": "s1", "ts": 4.0, "tool": "Edit", "ms": 1}, + ] + rows = bdd.build_top_slow(events, 4) + self.assertEqual(len(rows), 4) + # 非数字/缺失的 ms 一律按 0 计，不会把字符串 "120" 排到最前面。 + self.assertEqual(sum(r["ms"] for r in rows), 1) + + def test_disabled_by_default_prints_nothing_extra(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + metrics = self._write_metrics(root) + out = root / "dashboard-data.json" + stdout = self._run_main([ + "--project-root", td, "--out", str(out), + "--metrics-path", str(metrics), "--state-path", str(root / ".state.json"), + ]) + self.assertIn("wrote ", stdout) + self.assertNotIn("top-slow", stdout) + self.assertEqual(self._printed_tools(stdout), []) + + def test_enabled_prints_tool_and_ms_descending(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + metrics = self._write_metrics(root) + out = root / "dashboard-data.json" + stdout = self._run_main([ + "--project-root", td, "--out", str(out), + "--metrics-path", str(metrics), "--state-path", str(root / ".state.json"), + "--top-slow", "2", + ]) + self.assertIn("wrote ", stdout) + self.assertIn("top-slow", stdout) + self.assertEqual(self._printed_tools(stdout), ["Bash", "Read"]) + self.assertIn("900ms", stdout) + self.assertNotIn("120ms", stdout) # N=2，第三名的 Grep 不该出现 + + def test_enabled_does_not_change_dashboard_data_structure(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + metrics = self._write_metrics(root) + state = root / ".state.json" + state.write_text(json.dumps({"s1": {"skills": {}}}), encoding="utf-8") + # 输出到两个不同文件再比对，避免"第二次跑覆盖了第一次"导致对比失真。 + out_on = root / "on.json" + out_off = root / "off.json" + self._run_main([ + "--project-root", td, "--out", str(out_on), + "--metrics-path", str(metrics), "--state-path", str(state), "--top-slow", "5", + ]) + self._run_main([ + "--project-root", td, "--out", str(out_off), + "--metrics-path", str(metrics), "--state-path", str(state), + ]) + data_on = json.loads(out_on.read_text("utf-8")) + data_off = json.loads(out_off.read_text("utf-8")) + data_on.pop("generated_at") + data_off.pop("generated_at") + self.assertEqual(data_on, data_off) + # 兜底断言：结果没有以任何形式渗进 JSON（不只是"结构相同"）。 + self.assertNotIn("topSlow", json.dumps(data_on)) + self.assertNotIn("top-slow", json.dumps(data_on)) + + +class BuildDailyAndSessionsTurnAttributionTests(unittest.TestCase): + """真实场景复现过：AgentLens（写 state.current_turn 的那套 tracing）关闭时， + tool/usage 事件自己的 turn_id 永远是 None，只有 user_prompt_submit 事件带真实 + turn_id。修复前所有没有 turn_id 的事件会被塌缩进同一个 "?" 占位桶，看起来 + 像"整个会话只有 1 个 turn"，哪怕实际发了好几轮 prompt。""" + + def _events(self): + return [ + {"event": "user_prompt_submit", "sid": "s1", "ts": 1.0, "turn_id": "t1", "agent": "main"}, + {"event": "tool", "sid": "s1", "ts": 2.0, "tool": "Bash", "ms": 100, "agent": "main"}, + {"event": "usage", "sid": "s1", "ts": 3.0, "tokens": {"input": 10, "output": 5}, "agent": "main"}, + {"event": "user_prompt_submit", "sid": "s1", "ts": 4.0, "turn_id": "t2", "agent": "main"}, + {"event": "tool", "sid": "s1", "ts": 5.0, "tool": "Read", "ms": 50, "agent": "main"}, + {"event": "tool", "sid": "s1", "ts": 6.0, "tool": "Edit", "ms": 80, "agent": "main"}, + ] + + def test_turns_reflect_prompt_boundaries_not_a_single_bucket(self): + daily, sessions = bdd.build_daily_and_sessions(self._events()) + self.assertEqual(len(sessions), 1) + sess = sessions[0] + self.assertEqual(sess["turns"], 2) + self.assertEqual(sess["toolCalls"], 3) + self.assertEqual(len(sess["timeline"]), 2) + turn1_tools = [e["tool"] for e in sess["timeline"][0]["events"] if e["kind"] == "tool"] + turn2_tools = [e["tool"] for e in sess["timeline"][1]["events"] if e["kind"] == "tool"] + self.assertEqual(turn1_tools, ["Bash"]) + self.assertEqual(turn2_tools, ["Read", "Edit"]) + + def test_events_before_first_prompt_fall_back_to_unknown_bucket(self): + events = [ + {"event": "tool", "sid": "s1", "ts": 1.0, "tool": "Bash", "ms": 100, "agent": "main"}, + {"event": "user_prompt_submit", "sid": "s1", "ts": 2.0, "turn_id": "t1", "agent": "main"}, + {"event": "tool", "sid": "s1", "ts": 3.0, "tool": "Read", "ms": 50, "agent": "main"}, + ] + _, sessions = bdd.build_daily_and_sessions(events) + sess = sessions[0] + # 没有归属到任何真实 turn 的事件仍然单独成桶，不会被错误地并入第一个真实 turn。 + self.assertEqual(len(sess["timeline"]), 2) + self.assertEqual(sess["timeline"][0]["turn"], 1) + + def test_prompt_only_turn_with_no_tool_calls_still_appears_in_timeline(self): + """真实数据复现过：纯对话轮次（用户发了 prompt，但助手没有调用任何工具， + 也没有产生 usage 事件）之前完全不会在 timeline 里建桶——turns 计数是对的 + （来自 user_prompt_submit 自带的 turn_id），但展开的时间线分组数会比 turns + 少，二者对不上。""" + events = [ + {"event": "user_prompt_submit", "sid": "s1", "ts": 1.0, "turn_id": "t1", "agent": "main"}, + {"event": "tool", "sid": "s1", "ts": 2.0, "tool": "Read", "ms": 100, "agent": "main"}, + {"event": "user_prompt_submit", "sid": "s1", "ts": 3.0, "turn_id": "t2", "agent": "main"}, + {"event": "user_prompt_submit", "sid": "s1", "ts": 4.0, "turn_id": "t3", "agent": "main"}, + {"event": "tool", "sid": "s1", "ts": 5.0, "tool": "Grep", "ms": 50, "agent": "main"}, + ] + _, sessions = bdd.build_daily_and_sessions(events) + sess = sessions[0] + self.assertEqual(sess["turns"], 3) + self.assertEqual(len(sess["timeline"]), 3) + self.assertEqual(sess["timeline"][1]["events"], []) + + def test_prompt_only_session_still_counted_in_daily_session_count(self): + """真实数据复现过（TC5 当天 5 个 session，"总览"页汇总出的会话数却是 4）： + daily["_sids"]（"会话浏览"总览卡片"会话数"的数据源）之前只在 tool/usage + 分支里 add，一个全程只发了 prompt、没有触发任何工具调用也没有 usage 事件 + 的 session（比如用户发了消息但被拒绝/打断，没有真正执行）会被这天的 + sessionCount 完全漏掉——尽管 sessions 列表（"会话浏览"tab）里它确实在， + 导致总览页"会话数"比实际能展开看到的会话数少。""" + events = [ + {"event": "user_prompt_submit", "sid": "s1", "ts": 1.0, "turn_id": "t1", "agent": "main"}, + {"event": "tool", "sid": "s1", "ts": 2.0, "tool": "Read", "ms": 50, "agent": "main"}, + {"event": "user_prompt_submit", "sid": "s2", "ts": 3.0, "turn_id": "t1", "agent": "main"}, + ] + daily, sessions = bdd.build_daily_and_sessions(events) + self.assertEqual(len(sessions), 2) + self.assertEqual(len(daily), 1) + self.assertEqual(daily[0]["sessionCount"], 2) + + +class BuildDevflowRunsClassicSoloTests(unittest.TestCase): + """真实运行验证过：Classic small 任务会经过 PHASE-0 -> SOLO -> TASK-05 + （knowledge 由 solo-developer 合并执行），不是只有 PHASE-0/SOLO 两步。""" + + def test_solo_run_includes_task05_and_marks_phase0_completed(self): + with tempfile.TemporaryDirectory() as td: + project_root = Path(td) + run_dir = project_root / "artifacts" / "solo-with-knowledge_20260914_0600" + run_dir.mkdir(parents=True) + (run_dir / "workflow-state.json").write_text(json.dumps({ + "version": "1.3", + "task_slug": "solo-with-knowledge_20260914_0600", + "size_class": "small", + "current_stage": "SUMMARY", + "last_event": "workflow_completed", + # 注意：真实 schema 里 stages{} 不含 PHASE-0 —— Phase 0 是隐式完成的。 + "stages": { + "SOLO": {"status": "completed", "executor": "solo-developer", "retry_count": 0}, + "TASK-02": {"status": "skipped", "executor": "architect", "retry_count": 0}, + "TASK-03": {"status": "skipped", "executor": "developer", "retry_count": 0}, + "CODE-REVIEW": {"status": "skipped", "executor": "code-reviewer", "retry_count": 0}, + "TASK-04": {"status": "skipped", "executor": "test-engineer", "retry_count": 0}, + "TASK-05": {"status": "completed", "executor": "solo-developer", "retry_count": 0}, + }, + }), encoding="utf-8") + + runs = bdd.build_devflow_runs(project_root) + self.assertEqual(len(runs), 1) + stage_by_key = {s["key"]: s["status"] for s in runs[0]["stages"]} + self.assertIn("TASK-05", stage_by_key) + self.assertEqual(stage_by_key["TASK-05"], "completed") + self.assertEqual(stage_by_key["PHASE-0"], "completed") + self.assertNotIn("TASK-01", stage_by_key) # SOLO 路径不该混进 medium/large 的阶段 + + +class BuildDevflowRunsMediumWithLingeringSoloKeyTests(unittest.TestCase): + """用户反馈复现：workflow-state.json 模板固定给每次运行都写一份 SOLO stage + （见 assets/workflow-state-template.json 里 SOLO.description 的说明： + "仅当 size_class==small 时启用，否则保持 skipped"），medium/large 任务从不会 + 真正执行它，状态停在 "pending" 或 "skipped"。修复前 `"SOLO" in stages` 只看 + key 存不存在，不看状态，导致这类 medium 任务被误判成 solo 路径，阶段视图会用 + CLASSIC_SOLO_ORDER 渲染，把 TASK-01/TASK-02/TASK-03/CODE-REVIEW/TASK-04 全部 + 从 stepper 里漏掉。""" + + def _run(self, solo_status: str): + with tempfile.TemporaryDirectory() as td: + project_root = Path(td) + run_dir = project_root / "artifacts" / "medium-with-lingering-solo_20260915_0900" + run_dir.mkdir(parents=True) + (run_dir / "workflow-state.json").write_text(json.dumps({ + "version": "1.3", + "task_slug": "medium-with-lingering-solo_20260915_0900", + "size_class": "medium", + "current_stage": "TASK-03", + "last_event": "TASK-02_completed", + "stages": { + "SOLO": {"status": solo_status, "executor": "solo-developer", "retry_count": 0}, + "TASK-01": {"status": "completed", "executor": "main", "retry_count": 0}, + "TASK-02": {"status": "completed", "executor": "architect", "retry_count": 0}, + "TASK-03": {"status": "in_progress", "executor": "developer", "retry_count": 0}, + "CODE-REVIEW": {"status": "pending", "executor": "code-reviewer", "retry_count": 0}, + "TASK-04": {"status": "pending", "executor": "test-engineer", "retry_count": 0}, + "TASK-05": {"status": "pending", "executor": "knowledge-engineer", "retry_count": 0}, + }, + }), encoding="utf-8") + return bdd.build_devflow_runs(project_root) + + def test_medium_run_with_skipped_solo_key_uses_full_classic_order(self): + runs = self._run("skipped") + self.assertEqual(len(runs), 1) + stage_by_key = {s["key"]: s["status"] for s in runs[0]["stages"]} + # 修复前：这里会走 CLASSIC_SOLO_ORDER，TASK-01/02/03/CODE-REVIEW/TASK-04 全部消失。 + self.assertIn("TASK-01", stage_by_key) + self.assertIn("TASK-02", stage_by_key) + self.assertIn("TASK-03", stage_by_key) + self.assertIn("CODE-REVIEW", stage_by_key) + self.assertIn("TASK-04", stage_by_key) + self.assertEqual(stage_by_key["TASK-02"], "completed") + self.assertEqual(stage_by_key["TASK-03"], "in_progress") + + def test_medium_run_with_pending_solo_key_uses_full_classic_order(self): + # 模板初始状态是 "pending"（不是所有实现都会显式改成 "skipped"），同样不该被判成 solo。 + runs = self._run("pending") + stage_by_key = {s["key"]: s["status"] for s in runs[0]["stages"]} + self.assertIn("TASK-01", stage_by_key) + self.assertIn("CODE-REVIEW", stage_by_key) + + +class BuildDevflowRunsDurationTests(unittest.TestCase): + """真实数据复现过：TASK-05 的 started_at 可能早于 SOLO 的 completed_at + （同一次 solo-developer 执行内部的子步骤，不是真正先后发生的两个阶段）。 + total_duration 必须按真实起止跨度算，不能对逐阶段 duration 求和——否则会把 + 重叠部分重复计入，虚高于源数据本身反映的运行时长（这次修复前是 600s， + 真实跨度只有 540s）。""" + + def _write_state(self, project_root: Path, stages: dict) -> None: + run_dir = project_root / "artifacts" / "overlap-run_20260914_0620" + run_dir.mkdir(parents=True) + (run_dir / "workflow-state.json").write_text(json.dumps({ + "version": "1.3", + "task_slug": "overlap-run_20260914_0620", + "size_class": "small", + "current_stage": "SUMMARY", + "last_event": "workflow_completed", + "stages": stages, + }), encoding="utf-8") + + def test_total_duration_uses_true_span_not_sum_of_overlapping_stages(self): + with tempfile.TemporaryDirectory() as td: + project_root = Path(td) + self._write_state(project_root, { + "SOLO": { + "status": "completed", "executor": "solo-developer", "retry_count": 0, + "started_at": "2026-09-14T06:20:00Z", "completed_at": "2026-09-14T06:29:00Z", + }, + "TASK-05": { + "status": "completed", "executor": "solo-developer", "retry_count": 0, + # 早于 SOLO 的 completed_at —— 真实数据里观测到的重叠场景。 + "started_at": "2026-09-14T06:28:00Z", "completed_at": "2026-09-14T06:29:00Z", + }, + }) + runs = bdd.build_devflow_runs(project_root) + self.assertEqual(len(runs), 1) + # 真实跨度是 06:20~06:29 = 540s，不是 540+60=600s。 + self.assertEqual(runs[0]["duration"], 540) + stage_by_key = {s["key"]: s["duration"] for s in runs[0]["stages"]} + # 单个阶段自身的 duration 不受影响，仍然如实反映各自的起止跨度。 + self.assertEqual(stage_by_key["SOLO"], 540) + self.assertEqual(stage_by_key["TASK-05"], 60) + + def test_total_duration_is_zero_when_no_stage_has_valid_timestamps(self): + with tempfile.TemporaryDirectory() as td: + project_root = Path(td) + self._write_state(project_root, { + "SOLO": {"status": "completed", "executor": "solo-developer", "retry_count": 0}, + "TASK-05": {"status": "completed", "executor": "solo-developer", "retry_count": 0}, + }) + runs = bdd.build_devflow_runs(project_root) + self.assertEqual(runs[0]["duration"], 0) + + +class BuildDailyAndSessionsPerEventAgentTests(unittest.TestCase): + """真实数据复现过（TC5，size_class=small 但走了真实 team_create/send_message + 派发）：同一个 sid 下，metrics.ndjson 的 tool/usage 事件本来就各自带着准确的 + agent 字段（一次真实会话里 66 条 main、64 条 solo-developer），但展开进 + timeline 的每条 event 字典只有 kind/tool/ms/err，从不写回 agent —— 时间线里 + 完全看不出某次工具调用到底是 main 自己做的还是派发给 solo-developer 后做的， + 等于白白丢弃了上游已经采集到的数据。""" + + def _events(self): + return [ + {"event": "user_prompt_submit", "sid": "s1", "ts": 1.0, "turn_id": "t1", "agent": "main"}, + {"event": "tool", "sid": "s1", "ts": 2.0, "tool": "Agent", "ms": 50, "agent": "main"}, + {"event": "tool", "sid": "s1", "ts": 3.0, "tool": "Bash", "ms": 100, "agent": "solo-developer"}, + {"event": "usage", "sid": "s1", "ts": 4.0, "tokens": {"input": 10, "output": 5}, "agent": "solo-developer"}, + {"event": "tool", "sid": "s1", "ts": 5.0, "tool": "SendMessage", "ms": 30, "agent": "main"}, + ] + + def test_timeline_events_carry_the_agent_that_actually_ran_them(self): + _, sessions = bdd.build_daily_and_sessions(self._events()) + sess = sessions[0] + events = sess["timeline"][0]["events"] + by_tool = {e.get("tool"): e for e in events if e["kind"] == "tool"} + self.assertEqual(by_tool["Agent"]["agent"], "main") + self.assertEqual(by_tool["Bash"]["agent"], "solo-developer") + self.assertEqual(by_tool["SendMessage"]["agent"], "main") + usage_events = [e for e in events if e["kind"] == "usage"] + self.assertEqual(usage_events[0]["agent"], "solo-developer") + + def test_missing_agent_on_raw_event_falls_back_to_main(self): + events = [ + {"event": "user_prompt_submit", "sid": "s1", "ts": 1.0, "turn_id": "t1"}, + {"event": "tool", "sid": "s1", "ts": 2.0, "tool": "Read", "ms": 10}, + ] + _, sessions = bdd.build_daily_and_sessions(events) + sess = sessions[0] + self.assertEqual(sess["timeline"][0]["events"][0]["agent"], "main") + + +class BuildDispatchTests(unittest.TestCase): + """真实数据复现过（TC5 的 dashboard-data-top-slow 会话，sid 01a09ef5...）： + 修复前 build_dispatch() 把 agent_history 里所有 evidence 以 "dispatch" 或 + "inbox" 开头的条目一律计数，导致两个真实问题—— + 1) solo-developer 被上报回声（inbox@report:*）重复计入，真实派发只有 2 次， + 却显示成 5； + 2) team-lead 显示成一个"被 main 派发的子 agent"，但 team-lead 在 + core/agent_identity.py::normalize_role_name 里本来就和 main 归为一类 + （它是 CodeBuddy 原生 team 基础设施里 lead session 自己的 mailbox 名， + 不是真实存在的子 agent），面板标题明明叫"Main → 子 Agent 派发次数"， + 混进一个其实等价于 main 自己的假子 agent。""" + + def _real_tc5_agent_history(self): + # 逐条取自真实会话 01a09ef5-1c6b-7928-870e-a5056ff360e1 的 .state.json。 + return [ + {"ts": 1.0, "agent": "solo-developer", "evidence": "dispatch@Agent<-main"}, + {"ts": 2.0, "agent": "team-lead", "evidence": "dispatch@SendMessage<-solo-developer"}, + {"ts": 3.0, "agent": "solo-developer", "evidence": "inbox@report:solo-developer"}, + {"ts": 4.0, "agent": "solo-developer", "evidence": "inbox@dispatch:solo-developer"}, + {"ts": 5.0, "agent": "main", "evidence": "dispatch@SendMessage<-solo-developer"}, + {"ts": 6.0, "agent": "main", "evidence": "inbox@handoff:solo-developer->main"}, + {"ts": 7.0, "agent": "solo-developer", "evidence": "dispatch@SendMessage<-main"}, + {"ts": 8.0, "agent": "solo-developer", "evidence": "inbox@report:solo-developer"}, + ] + + def test_real_tc5_history_excludes_team_lead_and_report_echoes(self): + state = {"01a09ef5": {"agent_history": self._real_tc5_agent_history()}} + rows = bdd.build_dispatch(state) + # team-lead（main 的别名）完全不出现；solo-developer 只数真正代表新派发的 + # 3 条证据（2 条工具触发的 dispatch@ + 1 条 inbox 独立确认的 dispatch:）， + # 2 条上报回声（inbox@report:*）不计入。 + self.assertEqual(rows, [{"agent": "solo-developer", "count": 3}]) + + def test_inbox_dispatch_evidence_alone_is_still_counted(self): + """inbox 扫描独立确认的 "main 派给了谁"（inbox@dispatch:*）是原生 team + 基础设施里唯一能感知到、但没有经过 Agent/Task 工具调用拦截到的派发方式 + （例如 team-lead 用 mailbox 直接投递任务），必须保留，不能因为过滤 + report/handoff 回声就连这类真实派发信号也一起丢掉。""" + state = {"s1": {"agent_history": [ + {"ts": 1.0, "agent": "architect", "evidence": "inbox@dispatch:architect"}, + ]}} + rows = bdd.build_dispatch(state) + self.assertEqual(rows, [{"agent": "architect", "count": 1}]) + + def test_report_and_handoff_evidence_alone_are_not_dispatches(self): + state = {"s1": {"agent_history": [ + {"ts": 1.0, "agent": "developer", "evidence": "inbox@report:developer"}, + {"ts": 2.0, "agent": "test-engineer", "evidence": "inbox@handoff:developer->test-engineer"}, + ]}} + rows = bdd.build_dispatch(state) + self.assertEqual(rows, []) + + def test_team_lead_is_excluded_even_without_report_noise(self): + state = {"s1": {"agent_history": [ + {"ts": 1.0, "agent": "team-lead", "evidence": "dispatch@SendMessage<-main"}, + ]}} + rows = bdd.build_dispatch(state) + self.assertEqual(rows, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/.cursor/skills/agent-observability/tests/test_cls_sink.py b/.cursor/skills/agent-observability/tests/test_cls_sink.py new file mode 100644 index 0000000..2da9d6a --- /dev/null +++ b/.cursor/skills/agent-observability/tests/test_cls_sink.py @@ -0,0 +1,184 @@ +from __future__ import annotations + +import json +import os +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "scripts" +if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) + +from core import cls_sink, emitter # type: ignore + + +class CLSSinkTests(unittest.TestCase): + def _write_cls_env(self, root: Path, *extra_lines: str) -> None: + lines = [ + "CLS_TOPIC_ID=test-topic-id", + "CLS_ENDPOINT=cls.internal.tencentcloudapi.com", + "CLS_SERVICE_NAME=agent-observability", + "CLS_UPLOAD_TIMEOUT_SECONDS=20", + *extra_lines, + ] + (root / ".env").write_text("\n".join(lines) + "\n", encoding="utf-8") + + def test_load_config_supports_cls_and_tc_credentials(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + self._write_cls_env( + root, + "TC_SECRET_ID=test-secret-id", + "TC_SECRET_KEY=test-secret-key", + ) + with mock.patch.dict(os.environ, {}, clear=True): + config = cls_sink.load_config(td) + + self.assertTrue(config.enabled) + self.assertEqual(config.endpoint, "cls.internal.tencentcloudapi.com") + self.assertEqual(config.topic_id, "test-topic-id") + self.assertEqual(config.secret_id, "test-secret-id") + self.assertEqual(config.secret_key, "test-secret-key") + self.assertEqual(config.secret_token, "") + self.assertEqual(config.service_name, "agent-observability") + self.assertEqual(config.timeout_seconds, 20) + self.assertTrue(config.ready) + + def test_load_config_supports_tencentcloud_scoped_credentials(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + self._write_cls_env(root) + with mock.patch.dict( + os.environ, + { + "TENCENTCLOUD_SECRET_ID_438167613": "scoped-secret-id", + "TENCENTCLOUD_SECRET_KEY_438167613": "scoped-secret-key", + }, + clear=True, + ): + config = cls_sink.load_config(td) + + self.assertEqual(config.secret_id, "scoped-secret-id") + self.assertEqual(config.secret_key, "scoped-secret-key") + self.assertEqual(config.timeout_seconds, 20) + self.assertTrue(config.ready) + + def test_mirror_record_invokes_node_uploader(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + self._write_cls_env( + root, + "CLS_SECRET_ID=test-secret-id", + "CLS_SECRET_KEY=test-secret-key", + ) + with mock.patch.object(cls_sink, "_put_logs_via_api", return_value=(False, "python-failed")), \ + mock.patch.object(cls_sink.subprocess, "run", return_value=mock.Mock(returncode=0, stdout="", stderr="")) as run, \ + mock.patch.dict(os.environ, {}, clear=True): + ok = cls_sink.mirror_record({"event": "tool", "sid": "s-1"}, cwd=td) + + self.assertTrue(ok) + run.assert_called_once() + payload = json.loads(run.call_args.kwargs["input"]) + self.assertEqual(payload["records"][0]["event"], "tool") + self.assertEqual(payload["topicId"], "test-topic-id") + self.assertEqual(payload["secretToken"], "") + + def test_load_config_requires_credentials_from_env(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + self._write_cls_env(root) + with mock.patch.dict(os.environ, {}, clear=True): + config = cls_sink.load_config(td) + + self.assertEqual(config.secret_id, "") + self.assertEqual(config.secret_key, "") + self.assertEqual(config.secret_token, "") + self.assertEqual(config.timeout_seconds, 20) + self.assertFalse(config.ready) + + def test_load_config_reads_secret_token_aliases(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + self._write_cls_env( + root, + "CLS_SECRET_ID=test-secret-id", + "CLS_SECRET_KEY=test-secret-key", + "TC_SESSION_TOKEN=test-session-token", + ) + with mock.patch.dict(os.environ, {}, clear=True): + config = cls_sink.load_config(td) + + self.assertEqual(config.secret_token, "test-session-token") + + def test_load_config_supports_timeout_override(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + self._write_cls_env( + root, + "CLS_SECRET_ID=test-secret-id", + "CLS_SECRET_KEY=test-secret-key", + "CLS_UPLOAD_TIMEOUT_SECONDS=45", + ) + with mock.patch.dict(os.environ, {}, clear=True): + config = cls_sink.load_config(td) + + self.assertEqual(config.timeout_seconds, 45) + + def test_mirror_record_writes_debug_when_uploader_fails(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + debug_path = root / "cls-push-debug.ndjson" + self._write_cls_env( + root, + "CLS_SECRET_ID=test-secret-id", + "CLS_SECRET_KEY=test-secret-key", + ) + with mock.patch.object(cls_sink, "debug_log_path", return_value=debug_path), \ + mock.patch.object(cls_sink, "_put_logs_via_api", return_value=(False, "python-failed")), \ + mock.patch.object(cls_sink.subprocess, "run", return_value=mock.Mock(returncode=1, stdout="", stderr="boom")), \ + mock.patch.dict(os.environ, {}, clear=True): + ok = cls_sink.mirror_record({"event": "tool", "sid": "s-2"}, cwd=td) + + entries = [json.loads(line) for line in debug_path.read_text("utf-8").splitlines() if line.strip()] + + self.assertFalse(ok) + self.assertEqual(entries[-1]["stage"], "uploader_failed") + self.assertEqual(entries[-1]["stderr"], "boom") + self.assertEqual(entries[-1]["timeout_seconds"], 20) + + def test_mirror_record_writes_success_debug_when_uploader_succeeds(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + debug_path = root / "cls-push-debug.ndjson" + self._write_cls_env( + root, + "CLS_SECRET_ID=test-secret-id", + "CLS_SECRET_KEY=test-secret-key", + ) + with mock.patch.object(cls_sink, "debug_log_path", return_value=debug_path), \ + mock.patch.object(cls_sink, "_put_logs_via_api", return_value=(True, "ok")), \ + mock.patch.dict(os.environ, {}, clear=True): + ok = cls_sink.mirror_record({"event": "tool", "sid": "s-3"}, cwd=td) + + entries = [json.loads(line) for line in debug_path.read_text("utf-8").splitlines() if line.strip()] + + self.assertTrue(ok) + self.assertEqual(entries[0]["stage"], "uploader_start") + self.assertEqual(entries[-1]["stage"], "uploader_ok") + self.assertEqual(entries[-1]["method"], "python_api_v3") + + def test_emit_keeps_local_log_when_cls_push_raises(self): + with tempfile.TemporaryDirectory() as td: + log_path = Path(td) / "metrics.ndjson" + with mock.patch.object(emitter.cls_sink, "mirror_record", side_effect=RuntimeError("boom")): + emitter.emit(log_path, {"event": "start", "sid": "s-emit"}) + + records = [json.loads(line) for line in log_path.read_text("utf-8").splitlines() if line.strip()] + + self.assertEqual(len(records), 1) + self.assertEqual(records[0]["event"], "start") + self.assertEqual(records[0]["sid"], "s-emit") diff --git a/.cursor/skills/agent-observability/tests/test_collector.py b/.cursor/skills/agent-observability/tests/test_collector.py new file mode 100644 index 0000000..05547f0 --- /dev/null +++ b/.cursor/skills/agent-observability/tests/test_collector.py @@ -0,0 +1,715 @@ +from __future__ import annotations + +import json +import tempfile +import time +import unittest +from pathlib import Path +import sys + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "scripts" +if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) + +from core import agent_identity, agentlens, collector, emitter, runtime, state as st # type: ignore + + +class CollectorTests(unittest.TestCase): + def test_cost_of_uses_known_model_price_table(self): + tokens = {"input": 1000, "output": 200, "cache_read": 0, "total": 1200} + cost = emitter.cost_of(tokens, "gpt-4o") + self.assertIsNotNone(cost) + self.assertGreaterEqual(cost, 0.0) + + def test_cost_of_returns_none_for_unknown_model(self): + tokens = {"input": 1000, "output": 200, "cache_read": 0, "total": 1200} + self.assertIsNone(emitter.cost_of(tokens, "unknown-model")) + + def test_emit_adds_codebuddy_cli_data_source(self): + with tempfile.TemporaryDirectory() as td: + log_path = Path(td) / "metrics.ndjson" + + emitter.emit(log_path, {"event": "start", "sid": "s-data-source"}) + + record = json.loads(log_path.read_text(encoding="utf-8").strip()) + self.assertEqual(record["data_source"], "codebuddy-cli") + + def test_emit_preserves_explicit_data_source(self): + with tempfile.TemporaryDirectory() as td: + log_path = Path(td) / "metrics.ndjson" + + emitter.emit(log_path, {"event": "start", "sid": "s-data-source", "data_source": "manual"}) + + record = json.loads(log_path.read_text(encoding="utf-8").strip()) + self.assertEqual(record["data_source"], "manual") + + def test_emit_session_duration_uses_first_log_ts_when_state_was_recreated(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + log_path = root / "metrics.ndjson" + state_path = root / ".state.json" + log_path.write_text( + json.dumps({"event": "start", "sid": "s-duration", "ts": 100.0}) + "\n", + encoding="utf-8", + ) + state_path.write_text( + json.dumps({"s-duration": {"started_at": 190.0}}, ensure_ascii=False), + encoding="utf-8", + ) + + emitter.emit(log_path, {"event": "usage", "sid": "s-duration", "ts": 200.0, "tokens": {"input": 1, "output": 2}}) + + records = [json.loads(line) for line in log_path.read_text(encoding="utf-8").splitlines()] + self.assertEqual(records[-1]["session_duration_sec"], 100.0) + + def test_emit_session_duration_starts_at_zero_without_prior_log_or_state(self): + with tempfile.TemporaryDirectory() as td: + log_path = Path(td) / "metrics.ndjson" + + emitter.emit(log_path, {"event": "start", "sid": "s-new", "ts": 100.0}) + + record = json.loads(log_path.read_text(encoding="utf-8").strip()) + self.assertEqual(record["session_duration_sec"], 0.0) + + def test_parse_transcript_tail_extracts_tokens(self): + transcript = ROOT / "tests" / "fixtures" / "transcript.jsonl" + out = collector.parse_transcript_tail(str(transcript), max_lines=20) + self.assertIsInstance(out, dict) + self.assertIsInstance(out.get("tokens"), dict) + self.assertEqual(out["tokens"].get("input_tokens"), 2240) + self.assertEqual(out["tokens"].get("output_tokens"), 120) + usage_records = out.get("usage_records") or [] + tool_records = out.get("tool_records") or [] + self.assertEqual(len(usage_records), 3) + self.assertEqual(usage_records[0]["tokens"].get("input_tokens"), 1820) + self.assertEqual(usage_records[-1]["tokens"].get("input_tokens"), 2240) + self.assertEqual(usage_records[0].get("message_id"), "msg-001") + self.assertEqual(usage_records[-1].get("message_id"), "msg-003") + self.assertLess(usage_records[0]["offset"], usage_records[-1]["offset"]) + self.assertEqual(len(tool_records), 8) + self.assertEqual(tool_records[0]["tool"], "read_file") + self.assertEqual(tool_records[0]["message_id"], "msg-001") + self.assertEqual(tool_records[0]["next_message_id"], "msg-002") + self.assertEqual(tool_records[-1]["tool"], "Bash") + self.assertEqual(tool_records[-1]["message_id"], "msg-002") + self.assertEqual(tool_records[-1]["next_message_id"], "msg-003") + + def test_realtime_usage_is_request_level_and_dedupes_across_phase(self): + transcript = ROOT / "tests" / "fixtures" / "transcript.jsonl" + with tempfile.TemporaryDirectory() as td: + state_path = Path(td) / ".state.json" + sid = "s-transcript" + + scan = collector.collect_transcript_entries( + state_path=state_path, + sid=sid, + transcript_path=str(transcript), + max_lines=20, + ) + entries = scan.get("entries") or [] + + self.assertEqual(len(entries), 3) + self.assertEqual(entries[0]["tokens"]["input"], 1820) + self.assertGreater(entries[0]["offset"], 0) + self.assertEqual(entries[0]["message_id"], "msg-001") + self.assertEqual(entries[1]["tokens"]["input"], 2100) + self.assertEqual(entries[1]["tokens"]["total"], 2158) + self.assertEqual(len(scan.get("tool_records") or []), 8) + + usage_key = collector.build_transcript_event_key( + kind="usage", + tool="Read", + entry=entries[0], + ) + stop_key = collector.build_transcript_event_key( + kind="stop", + tool="__stop__", + entry=entries[0], + ) + other_tool_key = collector.build_transcript_event_key( + kind="usage", + tool="Bash", + entry=entries[0], + ) + + self.assertEqual(usage_key, stop_key) + self.assertEqual(usage_key, other_tool_key) + self.assertTrue(st.claim_transcript_event(state_path, sid, usage_key, str(transcript))) + self.assertFalse(st.claim_transcript_event(state_path, sid, stop_key, str(transcript))) + + def test_find_current_tool_message_id_prefers_transcript_tool_records(self): + transcript = ROOT / "tests" / "fixtures" / "transcript.jsonl" + with tempfile.TemporaryDirectory() as td: + state_path = Path(td) / ".state.json" + sid = "s-tool-mid" + + read_mid = collector.find_current_tool_message_id( + state_path, + sid, + str(transcript), + "Read", + ) + bash_mid = collector.find_current_tool_message_id( + state_path, + sid, + str(transcript), + "Bash", + ) + + self.assertEqual(read_mid, "msg-002") + self.assertEqual(bash_mid, "msg-003") + + def test_resolve_subagent_transcript_alias_and_tool_context(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + sid = "session-123" + main = root / f"{sid}.jsonl" + bundle = root / sid + subagents = bundle / "subagents" + subagents.mkdir(parents=True) + alias_session_id = "subagent-session-1" + subagent = subagents / "agent-a.jsonl" + main.write_text("", encoding="utf-8") + subagent.write_text( + "\n".join( + [ + json.dumps( + { + "type": "message", + "sessionId": alias_session_id, + "providerData": {"agent": "Explore"}, + } + ), + json.dumps( + { + "type": "function_call", + "sessionId": alias_session_id, + "name": "Grep", + "callId": "call-1", + "providerData": { + "agent": "Explore", + "messageId": "msg-sub-001", + }, + } + ), + ] + ) + + "\n", + encoding="utf-8", + ) + + aliased_path = str(root / f"{alias_session_id}.jsonl") + resolved = collector.resolve_transcript_path_alias(sid, aliased_path) + self.assertEqual(resolved, str(subagent.resolve())) + + state_path = root / ".state.json" + context = collector.find_current_tool_context( + state_path=state_path, + sid=sid, + transcript_path=aliased_path, + tool_name="Grep", + ) + self.assertEqual( + context, + { + "message_id": "msg-sub-001", + "transcript_path": str(subagent.resolve()), + "agent": "explore", + "tool_details": {"call_id": "call-1"}, + }, + ) + + def test_find_current_tool_context_prefers_nearest_timestamp(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + transcript = root / "session.jsonl" + transcript.write_text( + "\n".join( + [ + json.dumps( + { + "timestamp": 1000, + "type": "function_call", + "name": "Bash", + "callId": "call-1", + "providerData": {"messageId": "msg-early"}, + } + ), + json.dumps( + { + "timestamp": 5000, + "type": "function_call", + "name": "Bash", + "callId": "call-2", + "providerData": {"messageId": "msg-late"}, + } + ), + ] + ) + + "\n", + encoding="utf-8", + ) + state_path = root / ".state.json" + + early = collector.find_current_tool_context( + state_path=state_path, + sid="s-nearest", + transcript_path=str(transcript), + tool_name="Bash", + event_ts=1.2, + ) + late = collector.find_current_tool_context( + state_path=state_path, + sid="s-nearest", + transcript_path=str(transcript), + tool_name="Bash", + event_ts=4.9, + ) + + self.assertEqual(early["message_id"], "msg-early") + self.assertEqual(late["message_id"], "msg-late") + + def test_find_current_tool_context_claims_distinct_call_ids_with_same_timestamp(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + transcript = root / "session.jsonl" + transcript.write_text( + "\n".join( + [ + json.dumps( + { + "timestamp": 5000, + "type": "function_call", + "name": "Bash", + "callId": "call-1", + "arguments": "{\"command\":\"echo one\"}", + "providerData": {"messageId": "msg-shared"}, + } + ), + json.dumps( + { + "timestamp": 5000, + "type": "function_call", + "name": "Bash", + "callId": "call-2", + "arguments": "{\"command\":\"echo two\"}", + "providerData": {"messageId": "msg-shared"}, + } + ), + json.dumps( + { + "timestamp": 5000, + "type": "function_call_result", + "name": "Bash", + "callId": "call-1", + "providerData": { + "messageId": "msg-shared", + "toolResult": {"content": "result one"}, + }, + } + ), + json.dumps( + { + "timestamp": 5000, + "type": "function_call_result", + "name": "Bash", + "callId": "call-2", + "providerData": { + "messageId": "msg-shared", + "toolResult": {"content": "result two"}, + }, + } + ), + ] + ) + + "\n", + encoding="utf-8", + ) + state_path = root / ".state.json" + + first = collector.find_current_tool_context( + state_path=state_path, + sid="s-claim-shared", + transcript_path=str(transcript), + tool_name="Bash", + event_ts=4.9, + claim=True, + ) + second = collector.find_current_tool_context( + state_path=state_path, + sid="s-claim-shared", + transcript_path=str(transcript), + tool_name="Bash", + event_ts=4.9, + claim=True, + ) + third = collector.find_current_tool_context( + state_path=state_path, + sid="s-claim-shared", + transcript_path=str(transcript), + tool_name="Bash", + event_ts=4.9, + claim=True, + ) + + self.assertEqual(first["message_id"], "msg-shared") + self.assertEqual(second["message_id"], "msg-shared") + self.assertNotEqual(first["tool_details"]["call_id"], second["tool_details"]["call_id"]) + self.assertIn("arguments", first["tool_details"]) + self.assertIn("result_content", first["tool_details"]) + self.assertEqual(third, {"duplicate": True}) + + def test_find_fallback_usage_event_prefers_latest_matching_usage(self): + tool_event = { + "tool": "Bash", + "agent": "main", + "transcript_path": "/tmp/demo.jsonl", + } + usage_events = [ + { + "agent": "main", + "tool": "Read", + "transcript_path": "/tmp/demo.jsonl", + "message_id": "msg-old", + "source_offset": 100, + }, + { + "agent": "main", + "tool": "Bash", + "transcript_path": "/tmp/demo.jsonl", + "message_id": "msg-current", + "source_offset": 200, + }, + ] + + matched = collector.find_fallback_usage_event(tool_event, usage_events) + self.assertIsNotNone(matched) + self.assertEqual(matched["message_id"], "msg-current") + + def test_find_current_tool_context_prefers_explicit_call_id_over_nearest_timestamp(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + transcript = root / "session.jsonl" + transcript.write_text( + "\n".join( + [ + json.dumps( + { + "timestamp": 1000, + "type": "function_call", + "name": "Bash", + "callId": "call-early", + "providerData": {"messageId": "msg-early"}, + } + ), + json.dumps( + { + "timestamp": 1001, + "type": "function_call_result", + "name": "Bash", + "callId": "call-early", + "providerData": { + "messageId": "msg-early", + "toolResult": {"content": "early result"}, + }, + } + ), + json.dumps( + { + "timestamp": 5000, + "type": "function_call", + "name": "Bash", + "callId": "call-late", + "providerData": {"messageId": "msg-late"}, + } + ), + json.dumps( + { + "timestamp": 5001, + "type": "function_call_result", + "name": "Bash", + "callId": "call-late", + "providerData": { + "messageId": "msg-late", + "toolResult": {"content": "late result"}, + }, + } + ), + ] + ) + + "\n", + encoding="utf-8", + ) + state_path = root / ".state.json" + + context = collector.find_current_tool_context( + state_path=state_path, + sid="s-call-id-priority", + transcript_path=str(transcript), + tool_name="Bash", + call_id="call-early", + event_ts=4.9, + ) + + self.assertEqual(context["message_id"], "msg-early") + self.assertEqual(context["tool_details"]["call_id"], "call-early") + self.assertEqual(context["tool_details"]["result_content"], "early result") + + def test_find_current_tool_context_uses_next_message_id_when_available(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + transcript = root / "session.jsonl" + transcript.write_text( + "\n".join( + [ + json.dumps( + { + "timestamp": 1000, + "type": "function_call", + "name": "Read", + "callId": "call-1", + "providerData": {"messageId": "msg-tool"}, + } + ), + json.dumps( + { + "timestamp": 1001, + "type": "function_call_result", + "name": "Read", + "callId": "call-1", + "providerData": { + "messageId": "msg-tool", + "toolResult": {"content": "missing"}, + }, + } + ), + json.dumps( + { + "timestamp": 2000, + "type": "assistant", + "providerData": {"messageId": "msg-next"}, + "content": "next step", + } + ), + ] + ) + + "\n", + encoding="utf-8", + ) + state_path = root / ".state.json" + + context = collector.find_current_tool_context( + state_path=state_path, + sid="s-next-mid", + transcript_path=str(transcript), + tool_name="Read", + call_id="call-1", + ) + + self.assertEqual(context["message_id"], "msg-next") + self.assertEqual(context["tool_details"]["original_message_id"], "msg-tool") + self.assertEqual(context["tool_details"]["next_message_id"], "msg-next") + self.assertTrue(context["tool_details"]["message_id_reassigned"]) + + def test_flush_pending_tool_events_emits_after_transcript_catches_up(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + state_path = root / ".state.json" + log_path = root / "metrics.ndjson" + transcript = root / "session.jsonl" + transcript.write_text("", encoding="utf-8") + sid = "s-pending" + + tool_event = { + "event": "tool", + "sid": sid, + "agent": "main", + "tool": "Bash", + "ms": 42, + "transcript_path": str(transcript), + "turn_id": "turn-1", + "ts": 1.2, + "skill": [], + "rule": [], + "cwd": str(ROOT), + } + st.append_pending_tool_emit(state_path, sid, tool_event) + + transcript.write_text( + json.dumps( + { + "timestamp": 1300, + "type": "function_call", + "name": "Bash", + "callId": "call-1", + "providerData": {"messageId": "msg-later"}, + } + ) + + "\n", + encoding="utf-8", + ) + + emitted_calls: list[dict[str, object]] = [] + original_emit_post_step = agentlens.emit_post_step + try: + agentlens.emit_post_step = lambda **kwargs: emitted_calls.append(kwargs) + runtime.flush_pending_tool_events( + state_path=state_path, + log_path=log_path, + sid=sid, + ) + finally: + agentlens.emit_post_step = original_emit_post_step + + self.assertEqual(st.get_pending_tool_emits(state_path, sid), []) + lines = [json.loads(line) for line in log_path.read_text(encoding="utf-8").splitlines() if line.strip()] + self.assertEqual(len(lines), 1) + self.assertEqual(lines[0]["message_id"], "msg-later") + self.assertEqual(len(emitted_calls), 1) + self.assertEqual(emitted_calls[0]["tool_event"]["message_id"], "msg-later") + + def test_related_transcript_paths_include_subagents(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + sid = "session-123" + main = root / f"{sid}.jsonl" + bundle = root / sid + subagents = bundle / "subagents" + subagents.mkdir(parents=True) + sub_a = subagents / "agent-a.jsonl" + sub_b = subagents / "agent-b.jsonl" + for path in (main, sub_a, sub_b): + path.write_text("", encoding="utf-8") + + paths = collector.related_transcript_paths(sid, str(main)) + + self.assertEqual( + paths, + [ + str(main.resolve()), + str(sub_a.resolve()), + str(sub_b.resolve()), + ], + ) + + def test_subagent_transcript_maps_to_role_name(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + subagent = root / "subagents" / "agent-27e41af0.jsonl" + subagent.parent.mkdir(parents=True) + subagent.write_text( + '{"content":[{"text":""}]}\n', + encoding="utf-8", + ) + + self.assertEqual( + agent_identity.agent_for_transcript_path(str(subagent), "main"), + "knowledge-engineer", + ) + + def test_subagent_transcript_prefers_assignment_role_over_general_provider_agent(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + subagent = root / "subagents" / "agent-165e361f.jsonl" + subagent.parent.mkdir(parents=True) + subagent.write_text( + '{"type":"message","providerData":{"agent":"general-purpose"},"content":[{"text":"\\n待命。\\n你在本 devflow 中的角色：developer（开发角色）"}]}\n', + encoding="utf-8", + ) + + self.assertEqual( + agent_identity.agent_for_transcript_path(str(subagent), "main"), + "developer", + ) + + def test_inbox_standby_assignment_does_not_switch_current_agent(self): + messages = [ + { + "mailbox_name": "architect", + "mailbox_role": "architect", + "from_role": "main", + "summary": "Initial task assignment for architect", + "text": "待命，监听 main 唤醒。", + "payload": None, + "is_shutdown": False, + "is_standby": True, + } + ] + current, dispatched, meta = agent_identity.infer_identity_from_messages(messages) + self.assertIsNone(current) + self.assertIsNone(dispatched) + self.assertEqual(meta["messages_seen"], 1) + + def test_record_pre_post_duration(self): + with tempfile.TemporaryDirectory() as td: + pending = Path(td) / ".pending.json" + pre = {"session_id": "s1", "tool_name": "read_file"} + post = {"session_id": "s1", "tool_name": "read_file"} + collector.record_pre(pending, pre) + time.sleep(0.02) + ms = collector.record_post(pending, post) + self.assertIsInstance(ms, int) + self.assertGreaterEqual(ms, 0) + + def test_record_tool_usage_counts_skill_and_rule(self): + with tempfile.TemporaryDirectory() as td: + state_path = Path(td) / ".state.json" + sid = "s-tool" + # 先确保 session 结构 + s = st.load_state(state_path) + st.ensure_session(s, sid) + st.save_state(state_path, s) + + data = { + "tool_name": "use_skill", + "tool_input": {"command": "pdf", "filePath": "README.md"}, + } + skills_meta = {"pdf": {"source": "user", "version": "1.0.0"}} + rules_meta = { + "security": {"source": "project", "alwaysApply": True, "enabled": True, "globs": []} + } + collector.record_tool_usage( + state_path=state_path, + sid=sid, + data=data, + skills_meta=skills_meta, + rules_meta=rules_meta, + active_agent="main", + collect_skills=True, + ) + + skills_usage, rules_usage = collector.get_session_usage(state_path, sid) + self.assertEqual(skills_usage.get("pdf", {}).get("count"), 1) + self.assertEqual(rules_usage.get("security", {}).get("count"), 1) + + def test_step_span_registry_is_idempotent(self): + with tempfile.TemporaryDirectory() as td: + state_path = Path(td) / ".state.json" + sid = "s-step" + carrier = {"traceparent": "00-" + ("1" * 32) + "-" + ("2" * 16) + "-01"} + st.begin_turn(state_path, sid, "turn-1", carrier) + + first = st.upsert_step_span( + state_path, + sid, + "main", + "msg-001", + {"traceparent": "00-" + ("1" * 32) + "-" + ("3" * 16) + "-01"}, + transcript_path="/tmp/demo.jsonl", + ) + second = st.upsert_step_span( + state_path, + sid, + "main", + "msg-001", + {"traceparent": "00-" + ("1" * 32) + "-" + ("4" * 16) + "-01"}, + transcript_path="/tmp/demo.jsonl", + ) + + self.assertEqual(first, second) + self.assertEqual(st.get_step_span_carrier(state_path, sid, "main", "msg-001"), first) + + +if __name__ == "__main__": + unittest.main() diff --git a/.cursor/skills/agent-observability/tests/test_devflow.py b/.cursor/skills/agent-observability/tests/test_devflow.py new file mode 100644 index 0000000..9ad0a7f --- /dev/null +++ b/.cursor/skills/agent-observability/tests/test_devflow.py @@ -0,0 +1,389 @@ +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path +import sys + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "scripts" +if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) + +from core import agent_identity, devflow, state as st # type: ignore + + +def _write_team_config(team_dir: Path, *, lead_session_id: str, member_cwd: str) -> None: + team_dir.mkdir(parents=True, exist_ok=True) + (team_dir / "config.json").write_text( + json.dumps({ + "leadSessionId": lead_session_id, + "createdAt": 1, + "members": [{"cwd": member_cwd}], + }), + encoding="utf-8", + ) + + +class TaskSlugFromTeamDirTests(unittest.TestCase): + def test_strips_fixed_prefix(self): + team_dir = Path("/tmp/.codebuddy/teams/multi-agents-devflow-my-task_20260911_1200") + self.assertEqual(devflow.task_slug_from_team_dir(team_dir), "my-task_20260911_1200") + + def test_returns_none_for_non_devflow_team_dir(self): + team_dir = Path("/tmp/.codebuddy/teams/some-other-team") + self.assertIsNone(devflow.task_slug_from_team_dir(team_dir)) + + +class StageSnapshotTests(unittest.TestCase): + def test_projects_expected_fields_and_ignores_non_dict_stage(self): + workflow_state = { + "current_stage": "TASK-03", + "size_class": "medium", + "run_mode": "auto", + "stages": { + "TASK-02": {"status": "completed", "executor": "architect", "retry_count": 0, "review_result": None}, + "CODE-REVIEW": {"status": "failed", "executor": "code-reviewer", "retry_count": 1, "review_result": "failed"}, + "garbage": "not-a-dict", + }, + } + snap = devflow.stage_snapshot(workflow_state) + self.assertEqual(snap["current_stage"], "TASK-03") + self.assertEqual(snap["size_class"], "medium") + self.assertNotIn("garbage", snap["stages"]) + self.assertEqual(snap["stages"]["CODE-REVIEW"]["retry_count"], 1) + self.assertEqual(snap["stages"]["CODE-REVIEW"]["review_result"], "failed") + + def test_handles_missing_or_malformed_input(self): + self.assertEqual(devflow.stage_snapshot(None), {}) + self.assertEqual(devflow.stage_snapshot({"stages": "not-a-dict"}), { + "current_stage": None, "size_class": None, "run_mode": None, + "schema_version": None, "execution_mode": None, "host_adapter": None, "run_id": None, + "stages": {}, + }) + + +class DiffStageChangesTests(unittest.TestCase): + def test_first_observation_produces_no_changes(self): + curr = devflow.stage_snapshot({"stages": {"TASK-02": {"status": "completed", "retry_count": 0}}}) + self.assertEqual(devflow.diff_stage_changes(None, curr), []) + + def test_detects_retry_count_increase_as_a_change(self): + prev = devflow.stage_snapshot({"stages": {"CODE-REVIEW": {"status": "in_progress", "retry_count": 0}}}) + curr = devflow.stage_snapshot({"stages": {"CODE-REVIEW": {"status": "failed", "retry_count": 1, "review_result": "failed"}}}) + changes = devflow.diff_stage_changes(prev, curr) + self.assertEqual(len(changes), 1) + self.assertEqual(changes[0]["stage"], "CODE-REVIEW") + self.assertEqual(changes[0]["retry_count"], 1) + self.assertEqual(changes[0]["review_result"], "failed") + + def test_unchanged_stage_produces_no_entry(self): + snap = devflow.stage_snapshot({"stages": {"TASK-02": {"status": "completed", "retry_count": 0}}}) + self.assertEqual(devflow.diff_stage_changes(snap, snap), []) + + +class PtIdFromTranscriptPathTests(unittest.TestCase): + def test_extracts_pt_track_from_subagent_transcript_name(self): + path = "/tmp/session-123/subagents/sub-developer-PT-01.jsonl" + self.assertEqual(agent_identity.pt_id_from_transcript_path(path), "PT-01") + + def test_returns_none_for_non_subagent_or_non_pt_transcript(self): + self.assertIsNone(agent_identity.pt_id_from_transcript_path("/tmp/session-123.jsonl")) + self.assertIsNone(agent_identity.pt_id_from_transcript_path("/tmp/session-123/subagents/architect.jsonl")) + + +class ResolveAndDiffTests(unittest.TestCase): + def test_returns_none_when_no_devflow_team_found(self): + with tempfile.TemporaryDirectory() as td: + state_path = Path(td) / ".state.json" + result = devflow.resolve_and_diff(state_path, "sid-1", "/tmp/some-project") + self.assertIsNone(result) + state = st.load_state(state_path) + self.assertIsNone(state["sid-1"]["_devflow"]["context"]) + + def test_late_appearing_devflow_run_is_discovered_on_a_later_call(self): + """真实 bug 回归测试：同一个长生命周期 sid，会话开始时探测不到 devflow + （此时既没有 team 目录也没有 artifacts/），之后才真正跑起 devflow + （比如通过 Agent 工具后台 spawn，而不是从一开始就是独立 team 成员 sid）。 + 早期的"没找到"不能被永久缓存，必须在 artifacts/ 出现之后的下一次调用里发现它。""" + with tempfile.TemporaryDirectory() as td: + project_dir = Path(td) / "project" + project_dir.mkdir() + state_path = project_dir / ".codebuddy" / "skills" / "agent-observability" / "logs" / ".state.json" + sid = "sid-long-lived" + + # 会话早期：还没有任何 devflow 痕迹。 + first = devflow.resolve_and_diff(state_path, sid, str(project_dir)) + self.assertIsNone(first) + + # 同一个 sid，会话中途才出现 devflow 产物（没有 team 目录，走 artifacts 扫描兜底）。 + artifacts_dir = project_dir / "artifacts" / "late-appearing-task_20260914_1200" + artifacts_dir.mkdir(parents=True) + (artifacts_dir / "workflow-state.json").write_text(json.dumps({ + "version": "1.3", "current_stage": "SOLO", "size_class": "small", + "stages": {"SOLO": {"status": "completed", "executor": "solo-developer", "retry_count": 0}}, + }), encoding="utf-8") + + second = devflow.resolve_and_diff(state_path, sid, str(project_dir)) + self.assertIsNotNone(second) + self.assertEqual(second["task_slug"], "late-appearing-task_20260914_1200") + + def test_end_to_end_with_fake_team_and_workflow_state(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + project_dir = root / "project" + project_dir.mkdir() + config_home = root / "codebuddy-home" + teams_root = config_home / "teams" + team_dir = teams_root / "multi-agents-devflow-fix-token-bypass_20260911_0900" + _write_team_config(team_dir, lead_session_id="sid-42", member_cwd=str(project_dir)) + + artifacts_dir = project_dir / "artifacts" / "fix-token-bypass_20260911_0900" + artifacts_dir.mkdir(parents=True) + workflow_state_path = artifacts_dir / "workflow-state.json" + workflow_state_path.write_text(json.dumps({ + "current_stage": "TASK-03", + "size_class": "medium", + "stages": { + "TASK-02": {"status": "completed", "executor": "architect", "retry_count": 0}, + "TASK-03": {"status": "in_progress", "executor": "developer", "retry_count": 0}, + }, + }), encoding="utf-8") + + import os + old_env = os.environ.get("CODEBUDDY_CONFIG_DIR") + os.environ["CODEBUDDY_CONFIG_DIR"] = str(config_home) + try: + state_path = project_dir / ".codebuddy" / "skills" / "agent-observability" / "logs" / ".state.json" + sid = "sid-42" + + # 第一次调用：发现 devflow 上下文，但因为是首次观测不产出 changes。 + first = devflow.resolve_and_diff(state_path, sid, str(project_dir)) + self.assertIsNotNone(first) + self.assertEqual(first["task_slug"], "fix-token-bypass_20260911_0900") + self.assertEqual(first["current_stage"], "TASK-03") + self.assertEqual(first["changes"], []) + + # workflow-state.json 更新：TASK-03 打回重试。 + workflow_state_path.write_text(json.dumps({ + "current_stage": "TASK-03", + "size_class": "medium", + "stages": { + "TASK-02": {"status": "completed", "executor": "architect", "retry_count": 0}, + "TASK-03": {"status": "failed", "executor": "developer", "retry_count": 1}, + }, + }), encoding="utf-8") + + second = devflow.resolve_and_diff(state_path, sid, str(project_dir)) + self.assertEqual(len(second["changes"]), 1) + self.assertEqual(second["changes"][0]["stage"], "TASK-03") + self.assertEqual(second["changes"][0]["retry_count"], 1) + self.assertEqual(second["changes"][0]["status"], "failed") + + # 没有变化时，第三次调用应该不再产出 changes。 + third = devflow.resolve_and_diff(state_path, sid, str(project_dir)) + self.assertEqual(third["changes"], []) + finally: + if old_env is None: + os.environ.pop("CODEBUDDY_CONFIG_DIR", None) + else: + os.environ["CODEBUDDY_CONFIG_DIR"] = old_env + + +class PortableSchemaStageSnapshotTests(unittest.TestCase): + """Portable（v2.0）用 `executor_role` 而不是 `executor`，且没有 `review_result`。""" + + def test_reads_executor_role_and_top_level_execution_context(self): + workflow_state = { + "version": "2.0", + "current_stage": "IMPLEMENT", + "size_class": "medium", + "execution_mode": "isolated", + "host_adapter": "codebuddy", + "run_id": "run-abc", + "stages": { + "DESIGN": {"status": "completed", "executor_role": "devflow-architect", "retry_count": 0}, + "IMPLEMENT": {"status": "failed", "executor_role": "devflow-developer", "retry_count": 1}, + }, + } + snap = devflow.stage_snapshot(workflow_state) + self.assertEqual(snap["schema_version"], "2.0") + self.assertEqual(snap["execution_mode"], "isolated") + self.assertEqual(snap["stages"]["DESIGN"]["executor"], "devflow-architect") + self.assertEqual(snap["stages"]["IMPLEMENT"]["retry_count"], 1) + self.assertIsNone(snap["stages"]["IMPLEMENT"]["review_result"]) + + def test_diff_detects_retry_on_portable_shape_same_as_classic(self): + prev = devflow.stage_snapshot({ + "version": "2.0", + "stages": {"IMPLEMENT": {"status": "in_progress", "executor_role": "devflow-developer", "retry_count": 0}}, + }) + curr = devflow.stage_snapshot({ + "version": "2.0", + "stages": {"IMPLEMENT": {"status": "failed", "executor_role": "devflow-developer", "retry_count": 1}}, + }) + changes = devflow.diff_stage_changes(prev, curr) + self.assertEqual(len(changes), 1) + self.assertEqual(changes[0]["stage"], "IMPLEMENT") + self.assertEqual(changes[0]["retry_count"], 1) + self.assertEqual(changes[0]["executor"], "devflow-developer") + + +class ArtifactsScanFallbackTests(unittest.TestCase): + """`topology: spawn` 宿主没有 team 目录，靠扫 artifacts/ 兜底发现 task_slug。""" + + def test_no_artifacts_dir_returns_none(self): + with tempfile.TemporaryDirectory() as td: + self.assertIsNone(devflow._scan_artifacts_for_active_run(td)) + + def test_ignores_non_devflow_json_and_picks_in_progress_run(self): + with tempfile.TemporaryDirectory() as td: + root = Path(td) + artifacts = root / "artifacts" + + # 不是 devflow 产物的 JSON（没有 stages 字段），不能被误当成一次运行。 + noise_dir = artifacts / "not-a-devflow-run" + noise_dir.mkdir(parents=True) + (noise_dir / "workflow-state.json").write_text(json.dumps({"hello": "world"}), encoding="utf-8") + + done_dir = artifacts / "finished-task_20260910_0900" + done_dir.mkdir(parents=True) + (done_dir / "workflow-state.json").write_text(json.dumps({ + "status": "completed", "stages": {"DESIGN": {"status": "completed"}}, + }), encoding="utf-8") + + active_dir = artifacts / "active-task_20260911_1000" + active_dir.mkdir(parents=True) + (active_dir / "workflow-state.json").write_text(json.dumps({ + "status": "in_progress", "stages": {"IMPLEMENT": {"status": "in_progress"}}, + }), encoding="utf-8") + + result = devflow._scan_artifacts_for_active_run(str(root)) + self.assertIsNotNone(result) + self.assertEqual(result["task_slug"], "active-task_20260911_1000") + self.assertIsNone(result["team_dir"]) + + def test_finished_detection_works_for_classic_schema_without_top_level_status(self): + """Classic（v1.3）没有顶层 status 字段，"跑完没跑完"只能看 last_event。""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) + artifacts = root / "artifacts" + + done_dir = artifacts / "classic-done_20260910_0900" + done_dir.mkdir(parents=True) + (done_dir / "workflow-state.json").write_text(json.dumps({ + "version": "1.3", "last_event": "workflow_completed", + "stages": {"SOLO": {"status": "completed"}}, + }), encoding="utf-8") + + active_dir = artifacts / "classic-active_20260911_1000" + active_dir.mkdir(parents=True) + (active_dir / "workflow-state.json").write_text(json.dumps({ + "version": "1.3", "last_event": "TASK-02_completed", + "stages": {"TASK-02": {"status": "completed"}, "TASK-03": {"status": "in_progress"}}, + }), encoding="utf-8") + + result = devflow._scan_artifacts_for_active_run(str(root)) + self.assertIsNotNone(result) + self.assertEqual(result["task_slug"], "classic-active_20260911_1000") + + def test_resolve_and_diff_switches_task_slug_when_a_newer_run_appears(self): + """真实 bug 回归测试：同一个长生命周期 sid 先后归属两次不同的 devflow 运行 + （没有真正 team_create 时的降级场景——第二次 `/start-devflow` 复用了同一个 + session）。第一次探测缓存下的 task_slug 不能在第二次运行开始后继续沿用； + 必须切换到新的那个，而且两个 task_slug 各自的 diff 历史不能互相污染。""" + with tempfile.TemporaryDirectory() as td: + project_dir = Path(td) / "project" + state_path = project_dir / ".codebuddy" / "skills" / "agent-observability" / "logs" / ".state.json" + sid = "sid-reused-across-two-runs" + + first_dir = project_dir / "artifacts" / "first-task_20260914_0900" + first_dir.mkdir(parents=True) + (first_dir / "workflow-state.json").write_text(json.dumps({ + "version": "1.3", "last_event": "workflow_completed", "current_stage": "SOLO", + "stages": {"SOLO": {"status": "completed", "retry_count": 0}}, + }), encoding="utf-8") + + first = devflow.resolve_and_diff(state_path, sid, str(project_dir)) + self.assertEqual(first["task_slug"], "first-task_20260914_0900") + + # 同一个 sid，第二次运行出现，且比第一次更新（mtime 更新）。 + import time + time.sleep(0.01) + second_dir = project_dir / "artifacts" / "second-task_20260914_1100" + second_dir.mkdir(parents=True) + (second_dir / "workflow-state.json").write_text(json.dumps({ + "version": "1.3", "last_event": "workflow_completed", "current_stage": "SOLO", + "stages": {"SOLO": {"status": "completed", "retry_count": 1}}, + }), encoding="utf-8") + + second = devflow.resolve_and_diff(state_path, sid, str(project_dir)) + self.assertEqual(second["task_slug"], "second-task_20260914_1100") + self.assertEqual(second["changes"], []) # 对 second 是首次观测，不产出变更 + + # 第一个 task_slug 的历史没有被污染：如果它重新变成"最新"（比如被再次修改）， + # 应该正确切回，并且不会把 second 的历史错当成 first 的基线。 + (first_dir / "workflow-state.json").write_text(json.dumps({ + "version": "1.3", "last_event": "workflow_completed", "current_stage": "SOLO", + "stages": {"SOLO": {"status": "completed", "retry_count": 1}}, + }), encoding="utf-8") + time.sleep(0.01) + first_dir.joinpath("workflow-state.json").touch() + + third = devflow.resolve_and_diff(state_path, sid, str(project_dir)) + self.assertEqual(third["task_slug"], "first-task_20260914_0900") + # first 上一次被观测到时 retry_count 还是 0，现在变成 1——应该被识别为变化， + # 而不是被 second 的快照历史污染成"首次观测"或者对不上的 diff。 + self.assertEqual(len(third["changes"]), 1) + self.assertEqual(third["changes"][0]["retry_count"], 1) + + def test_resolve_and_diff_end_to_end_via_artifacts_scan_no_team(self): + with tempfile.TemporaryDirectory() as td: + project_dir = Path(td) / "project" + artifacts_dir = project_dir / "artifacts" / "portable-task_20260911_1100" + artifacts_dir.mkdir(parents=True) + state_file = artifacts_dir / "workflow-state.json" + state_file.write_text(json.dumps({ + "version": "2.0", + "status": "in_progress", + "current_stage": "REVIEW", + "execution_mode": "isolated", + "stages": {"REVIEW": {"status": "in_progress", "executor_role": "devflow-code-reviewer", "retry_count": 0}}, + }), encoding="utf-8") + + import os + old_env = os.environ.get("CODEBUDDY_CONFIG_DIR") + os.environ["CODEBUDDY_CONFIG_DIR"] = str(Path(td) / "empty-codebuddy-home") + try: + state_path = project_dir / ".codebuddy" / "skills" / "agent-observability" / "logs" / ".state.json" + sid = "sid-spawn-1" + + first = devflow.resolve_and_diff(state_path, sid, str(project_dir)) + self.assertIsNotNone(first) + self.assertEqual(first["task_slug"], "portable-task_20260911_1100") + self.assertEqual(first["schema_version"], "2.0") + self.assertEqual(first["execution_mode"], "isolated") + self.assertEqual(first["changes"], []) + + state_file.write_text(json.dumps({ + "version": "2.0", + "status": "in_progress", + "current_stage": "REVIEW", + "execution_mode": "isolated", + "stages": {"REVIEW": {"status": "failed", "executor_role": "devflow-code-reviewer", "retry_count": 1}}, + }), encoding="utf-8") + + second = devflow.resolve_and_diff(state_path, sid, str(project_dir)) + self.assertEqual(len(second["changes"]), 1) + self.assertEqual(second["changes"][0]["retry_count"], 1) + self.assertEqual(second["changes"][0]["status"], "failed") + finally: + if old_env is None: + os.environ.pop("CODEBUDDY_CONFIG_DIR", None) + else: + os.environ["CODEBUDDY_CONFIG_DIR"] = old_env + + +if __name__ == "__main__": + unittest.main() diff --git a/.cursor/skills/agent-observability/tests/test_pricing_overrides.py b/.cursor/skills/agent-observability/tests/test_pricing_overrides.py new file mode 100644 index 0000000..75615a3 --- /dev/null +++ b/.cursor/skills/agent-observability/tests/test_pricing_overrides.py @@ -0,0 +1,394 @@ +"""自定义模型定价覆盖 + 看板未定价模型提示。 + +覆盖文件是**人手写的常驻配置**，而且加载失败要静默降级——这类"错了也不许 +报错"的代码最容易在半年后被悄悄改坏（比如有人顺手把 try/except 去掉、把合并 +改成整 key 覆盖、或让降级路径开始 print）。这里把字段级合并、四种/五种降级 +路径、缓存语义、未定价口径、看板聚合与"不重算历史成本"逐条钉住。 + +两个容易踩的隔离坑，本文件统一在 setUp/tearDown 处理： +1. `load_prices()` 带 `lru_cache`，不 `clear_price_cache()` 用例之间会串味； +2. `AOBS_PRICING_OVERRIDES_PATH` 是进程级环境变量，不还原会污染同进程里 + 其它测试模块（尤其是会真的去读默认覆盖文件的 hook 链路）。 + +真实日志对账（hy4-preview-ioa / hy3-ioa 的计数）不写进单测：日志会持续增长， +写死数字会变成 flaky 用例。那部分以人工冒烟的方式在 TASK-04 报告里记录。 +""" +from __future__ import annotations + +import contextlib +import io +import json +import os +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "scripts" +if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) + +import build_dashboard_data as bdd # type: ignore +from core import emitter # type: ignore + +OVERRIDES_ENV = "AOBS_PRICING_OVERRIDES_PATH" + + +class _OverridesTestCase(unittest.TestCase): + """统一处理环境变量与 `load_prices()` 缓存的隔离。""" + + def setUp(self): + self._env_backup = os.environ.get(OVERRIDES_ENV) + os.environ.pop(OVERRIDES_ENV, None) + emitter.clear_price_cache() + self._tmp = tempfile.TemporaryDirectory() + self.tmp = Path(self._tmp.name) + + def tearDown(self): + # 顺序要紧：先还原 env，再清缓存，最后清目录，避免留下一个指向已删除 + # 目录的环境变量给下一个用例。 + if self._env_backup is None: + os.environ.pop(OVERRIDES_ENV, None) + else: + os.environ[OVERRIDES_ENV] = self._env_backup + emitter.clear_price_cache() + self._tmp.cleanup() + + def use_overrides(self, payload, name: str = "overrides.json") -> Path: + """把 payload 写成覆盖文件并指向它（JSON 文本原样写入，便于构造非法输入）。""" + path = self.tmp / name + text = payload if isinstance(payload, str) else json.dumps(payload) + path.write_text(text, encoding="utf-8") + os.environ[OVERRIDES_ENV] = str(path) + emitter.clear_price_cache() + return path + + def pure_builtin(self) -> dict: + """"纯内置表"的唯一权威来源——不把 pricing.json 的内容抄进断言里， + 否则内置表一调整这里就假红；但内置表本身必须是非空的，否则下面的 + "降级后 == 纯内置表" 断言会退化成空表比空表。""" + builtin = emitter._load_builtin_prices() + self.assertTrue(builtin, "内置价格表不应为空，否则降级断言失去意义") + self.assertIn("gpt-4o", builtin) + return builtin + + +class OverridesPathResolutionTests(_OverridesTestCase): + """AC4：路径可覆盖。默认位置刻意放在 skills 树**外**—— + `scripts/build-classic-hosts.py` 会整棵同步 `.codebuddy/skills`，放树内会让 + 用户每次改价都产生一次 `--check` drift，并被复制进 .claude/.cursor 宿主包。""" + + def test_env_var_wins_over_default(self): + path = self.use_overrides({"zz-model": {"output": 1.0}}) + self.assertEqual(emitter.resolve_overrides_path(), path) + + def test_tilde_is_expanded(self): + os.environ[OVERRIDES_ENV] = "~/aobs-overrides-test.json" + emitter.clear_price_cache() + self.assertEqual(emitter.resolve_overrides_path(), Path.home() / "aobs-overrides-test.json") + + def test_unset_falls_back_to_default_path(self): + self.assertEqual(emitter.resolve_overrides_path(), emitter.DEFAULT_OVERRIDES_PATH) + + def test_blank_env_falls_back_to_default(self): + # 空字符串/纯空白都等同于"没设置"，不能变成指向 CWD 的相对路径。 + os.environ[OVERRIDES_ENV] = " " + emitter.clear_price_cache() + self.assertEqual(emitter.resolve_overrides_path(), emitter.DEFAULT_OVERRIDES_PATH) + + def test_default_path_is_outside_the_skills_tree(self): + default = emitter.DEFAULT_OVERRIDES_PATH + self.assertIsNotNone(default) + self.assertEqual( + default, + ROOT.parents[2] / ".codebuddy" / "agent-observability" / "pricing.overrides.json", + ) + self.assertNotIn("skills", default.parts[-3:]) + self.assertFalse(str(default).startswith(str(ROOT))) + + +class MergePricesTests(unittest.TestCase): + """AC2：字段级合并。整 key 覆盖被刻意否决过——漏写字段会把该字段按 0 计， + 静默把成本算没，风险高于收益。""" + + def test_explicit_field_overrides_builtin_while_others_are_kept(self): + merged = emitter.merge_prices( + {"gpt-4o": {"input": 2.5, "output": 10.0, "cache_read": 1.25}}, + {"gpt-4o": {"output": 99.0}}, + ) + self.assertEqual(merged["gpt-4o"]["output"], 99.0) + self.assertEqual(merged["gpt-4o"]["input"], 2.5) + self.assertEqual(merged["gpt-4o"]["cache_read"], 1.25) + + def test_new_model_can_be_added(self): + merged = emitter.merge_prices({"gpt-4o": {"output": 10.0}}, {"zz-new": {"output": 9.0}}) + self.assertEqual(merged["zz-new"], {"output": 9.0}) + + def test_merge_does_not_mutate_its_arguments(self): + base = {"gpt-4o": {"input": 2.5, "output": 10.0}} + overrides = {"gpt-4o": {"output": 99.0}} + merged = emitter.merge_prices(base, overrides) + merged["gpt-4o"]["input"] = 0.0 + # 内置表是模块级共享数据（lru_cache 之外还有调用方持有引用）， + # 被就地改过的话一次合并会污染整条 hook 链路。 + self.assertEqual(base["gpt-4o"], {"input": 2.5, "output": 10.0}) + self.assertEqual(overrides["gpt-4o"], {"output": 99.0}) + + def test_empty_or_none_overrides_return_base_copy(self): + base = {"gpt-4o": {"output": 10.0}} + self.assertEqual(emitter.merge_prices(base, None), base) + self.assertEqual(emitter.merge_prices(base, {}), base) + + def test_none_base_returns_overrides_only(self): + self.assertEqual(emitter.merge_prices(None, {"zz-new": {"output": 1.0}}), {"zz-new": {"output": 1.0}}) + + def test_non_dict_rows_are_ignored(self): + merged = emitter.merge_prices({"gpt-4o": {"output": 10.0}}, {"zz-bad": 9.0, "zz-ok": {"output": 2.0}}) + self.assertNotIn("zz-bad", merged) + self.assertEqual(merged["zz-ok"], {"output": 2.0}) + + +class LoadPricesOverrideTests(_OverridesTestCase): + """AC2 + AC3：加载与降级。""" + + def test_override_applies_field_level_and_adds_model(self): + self.use_overrides({"zz-custom": {"input": 1.0, "output": 9.0}, "gpt-4o": {"output": 99.0}}) + prices = emitter.load_prices() + self.assertEqual(prices["zz-custom"], {"input": 1.0, "output": 9.0}) + self.assertEqual(prices["gpt-4o"]["output"], 99.0) + self.assertEqual(prices["gpt-4o"]["input"], self.pure_builtin()["gpt-4o"]["input"]) + + def test_missing_file_degrades_to_builtin(self): + os.environ[OVERRIDES_ENV] = str(self.tmp / "does-not-exist.json") + emitter.clear_price_cache() + self.assertEqual(emitter.load_prices(), self.pure_builtin()) + + def test_invalid_json_degrades_to_builtin(self): + self.use_overrides('{"gpt-4o": {"output": 99.0}') + self.assertEqual(emitter.load_prices(), self.pure_builtin()) + + def test_top_level_non_dict_degrades_to_builtin(self): + self.use_overrides("[1, 2, 3]") + self.assertEqual(emitter.load_prices(), self.pure_builtin()) + + def test_non_numeric_field_is_skipped_but_siblings_apply(self): + self.use_overrides({"gpt-4o": {"output": "abc", "input": 3.0}}) + prices = emitter.load_prices() + # 写错一个字段不该把同模型其它字段、其它模型的正确覆盖一起丢掉。 + self.assertEqual(prices["gpt-4o"]["input"], 3.0) + self.assertEqual(prices["gpt-4o"]["output"], self.pure_builtin()["gpt-4o"]["output"]) + + def test_model_with_only_bad_fields_is_dropped_and_stays_unpriced(self): + """计划外决策（已在 code review 接受）：全坏字段的模型不落表。 + 若落表成 `{}`，它既算不出成本又不算"未定价"，看板上会变成一个查不到 + 原因的空洞；丢弃后至少能在"模型定价"提醒里暴露出来。""" + self.use_overrides({"zz-all-bad": {"output": "abc"}}) + prices = emitter.load_prices() + self.assertNotIn("zz-all-bad", prices) + self.assertTrue(emitter.is_unpriced("zz-all-bad")) + + def test_bool_is_not_treated_as_a_price(self): + # bool 是 int 的子类，float(True) == 1.0 会静默变成"单价 1 美元"。 + self.use_overrides({"gpt-4o": {"output": True}}) + prices = emitter.load_prices() + self.assertEqual(prices["gpt-4o"]["output"], self.pure_builtin()["gpt-4o"]["output"]) + + def test_every_degradation_path_is_silent(self): + """静默是刻意的：hook 每次都是新进程，一旦因格式问题 print/抛异常， + 整条 hook 链路都会变得不可用。""" + for payload in ('{"broken": ', "[1,2,3]", '{"gpt-4o": {"output": "abc"}}', '{"gpt-4o": 1}'): + with self.subTest(payload=payload): + self.use_overrides(payload) + out, err = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(out), contextlib.redirect_stderr(err): + prices = emitter.load_prices() + self.assertEqual(prices, self.pure_builtin()) + self.assertEqual(out.getvalue(), "") + self.assertEqual(err.getvalue(), "") + + def test_load_prices_is_cached_until_cleared(self): + path = self.use_overrides({"zz-cache-model": {"output": 1.0}}) + self.assertEqual(emitter.load_prices()["zz-cache-model"]["output"], 1.0) + path.write_text(json.dumps({"zz-cache-model": {"output": 2.0}}), encoding="utf-8") + # 未清缓存：同进程内仍是旧值（hook 是短命进程，这个行为是可接受的）。 + self.assertEqual(emitter.load_prices()["zz-cache-model"]["output"], 1.0) + emitter.clear_price_cache() + self.assertEqual(emitter.load_prices()["zz-cache-model"]["output"], 2.0) + + def test_clear_price_cache_picks_up_env_change(self): + self.assertTrue(hasattr(emitter.load_prices, "cache_clear")) + self.use_overrides({"zz-a": {"output": 1.0}}) + self.assertIn("zz-a", emitter.load_prices()) + self.use_overrides({"zz-b": {"output": 2.0}}, name="second.json") + emitter.clear_price_cache() + prices = emitter.load_prices() + self.assertIn("zz-b", prices) + self.assertNotIn("zz-a", prices) + + +class IsUnpricedTests(_OverridesTestCase): + """D4：未定价的唯一口径是 `lookup_price(...) is None`。 + `lookup_price` 会做"最长子串"模糊兜底，所以 `gpt-4o-2024-11-20` 能算出成本， + 不该出现在"建议补价"里——口径一旦漂移，用户会去给明明已经命中的模型补价。""" + + def test_unknown_model_is_unpriced(self): + self.assertTrue(emitter.is_unpriced("zz-totally-unknown")) + + def test_fuzzy_substring_match_is_not_unpriced(self): + self.assertFalse(emitter.is_unpriced("gpt-4o-2024-11-20")) + + def test_missing_or_blank_model_is_unpriced(self): + for model in (None, "", " "): + with self.subTest(model=model): + self.assertTrue(emitter.is_unpriced(model)) + + def test_override_makes_a_model_priced(self): + self.assertTrue(emitter.is_unpriced("zz-then-priced")) + self.use_overrides({"zz-then-priced": {"output": 9.0}}) + self.assertFalse(emitter.is_unpriced("zz-then-priced")) + + +class BuildUnpricedModelsTests(_OverridesTestCase): + """AC5：只看板**统计**，绝不回头重算 cost_usd（D3：覆盖只对新事件生效）。""" + + def _usage(self, model, ts=1.0): + return {"event": "usage", "sid": "s1", "ts": ts, "model": model, "tokens": {"input": 1, "output": 1}} + + def test_counts_only_unpriced_usage_events(self): + events = [ + self._usage("zz-unpriced-a", 1.0), + self._usage("zz-unpriced-a", 2.0), + self._usage("zz-unpriced-b", 3.0), + self._usage("gpt-4o-2024-11-20", 4.0), # 模糊命中，不算未定价 + {"event": "usage", "sid": "s1", "ts": 5.0, "tokens": {"input": 1}}, # model 缺失 + self._usage(" ", 6.0), # model 空白 + {"event": "tool", "sid": "s1", "ts": 7.0, "tool": "Bash", "ms": 1}, # 非 usage + ] + rows = bdd.build_unpriced_models(events) + self.assertEqual(rows, [ + {"name": "zz-unpriced-a", "usageEvents": 2}, + {"name": "zz-unpriced-b", "usageEvents": 1}, + ]) + + def test_empty_input_returns_empty_list(self): + self.assertEqual(bdd.build_unpriced_models([]), []) + + def test_sorted_by_event_count_desc_then_name(self): + events = ( + [self._usage("zz-b", float(i)) for i in range(2)] + + [self._usage("zz-a", 9.0)] + + [self._usage("zz-c", float(i)) for i in range(2)] + ) + rows = bdd.build_unpriced_models(events) + # 计数相同的 zz-b / zz-c 必须按名字定序，否则两次生成的快照无法逐字节比对。 + self.assertEqual([(r["name"], r["usageEvents"]) for r in rows], + [("zz-b", 2), ("zz-c", 2), ("zz-a", 1)]) + + def test_pricing_a_model_removes_it_from_the_list(self): + events = [self._usage("zz-unpriced-a", 1.0), self._usage("zz-unpriced-b", 2.0)] + self.assertEqual(len(bdd.build_unpriced_models(events)), 2) + self.use_overrides({"zz-unpriced-a": {"output": 9.0}}) + rows = bdd.build_unpriced_models(events) + self.assertEqual(rows, [{"name": "zz-unpriced-b", "usageEvents": 1}]) + + def test_emitter_import_failure_degrades_to_empty_list(self): + """目标机可能没装 emitter 的间接依赖；此时只该退化"未定价"这一项， + 看板仍要能出片。""" + with mock.patch.object(bdd, "em", None): + self.assertEqual(bdd.build_unpriced_models([self._usage("zz-unpriced-a")]), []) + + +class MainSnapshotTests(_OverridesTestCase): + """main() 级别：键位、空列表、不重算成本、无覆盖时与改动前一致（AC1）。""" + + def _events(self): + return [ + {"event": "user_prompt_submit", "sid": "s1", "ts": 1.0, "turn_id": "t1", "agent": "main"}, + {"event": "usage", "sid": "s1", "ts": 2.0, "model": "gpt-4o-2024-11-20", + "tokens": {"input": 100, "output": 10}, "cost_usd": 0.0035, "agent": "main"}, + {"event": "usage", "sid": "s1", "ts": 3.0, "model": "zz-unpriced-a", + "tokens": {"input": 100, "output": 10}, "agent": "main"}, + {"event": "usage", "sid": "s1", "ts": 4.0, "model": "zz-unpriced-a", + "tokens": {"input": 100, "output": 10}, "agent": "main"}, + ] + + def _run_main(self, root: Path, out_name: str) -> tuple[dict, str]: + metrics = root / "metrics.ndjson" + metrics.write_text("\n".join(json.dumps(e) for e in self._events()) + "\n", encoding="utf-8") + state = root / ".state.json" + state.write_text("{}", encoding="utf-8") + out = root / out_name + old_argv = sys.argv + sys.argv = ["build_dashboard_data.py", "--project-root", str(root), + "--out", str(out), "--metrics-path", str(metrics), "--state-path", str(state)] + buf = io.StringIO() + try: + with contextlib.redirect_stdout(buf): + rc = bdd.main() + finally: + sys.argv = old_argv + self.assertEqual(rc, 0) + self.assertIn("wrote ", buf.getvalue()) + return json.loads(out.read_text("utf-8")), buf.getvalue() + + def test_snapshot_exposes_unpriced_models_right_after_model_costs(self): + with tempfile.TemporaryDirectory() as td: + data, _ = self._run_main(Path(td), "out.json") + keys = list(data.keys()) + self.assertEqual(keys[keys.index("modelCosts") + 1], "unpricedModels") + self.assertEqual(data["unpricedModels"], [{"name": "zz-unpriced-a", "usageEvents": 2}]) + self.assertEqual([m["name"] for m in data["modelCosts"]], ["gpt-4o-2024-11-20"]) + + def test_no_unpriced_models_yields_empty_list_not_null(self): + with tempfile.TemporaryDirectory() as td: + self.use_overrides({"zz-unpriced-a": {"output": 9.0}}) + data, _ = self._run_main(Path(td), "out.json") + self.assertEqual(data["unpricedModels"], []) + + def test_override_never_recomputes_already_written_costs(self): + """D3 的回归护栏：给未定价模型补价后，只有 unpricedModels 会变， + modelCosts / daily / sessions 必须逐字节不变（历史成本不回溯）。""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) + before, _ = self._run_main(root, "before.json") + self.use_overrides({"zz-unpriced-a": {"output": 9.0}}) + after, _ = self._run_main(root, "after.json") + before.pop("generated_at") + after.pop("generated_at") + self.assertEqual(before["unpricedModels"], [{"name": "zz-unpriced-a", "usageEvents": 2}]) + self.assertEqual(after["unpricedModels"], []) + before.pop("unpricedModels") + after.pop("unpricedModels") + self.assertEqual(before, after) + + def test_broken_override_file_keeps_cli_at_exit_zero_and_silent(self): + """AC3 的 CLI 层护栏：覆盖文件坏掉时看板仍然出片（exit 0、stderr 为空）， + 而不是把整条构建链路搞挂。""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) + self.use_overrides('{"gpt-4o": {"output": 99.0}') # 故意截断的非法 JSON + err = io.StringIO() + with contextlib.redirect_stderr(err): + data, _ = self._run_main(root, "out.json") + self.assertEqual(err.getvalue(), "") + self.assertEqual(data["unpricedModels"], [{"name": "zz-unpriced-a", "usageEvents": 2}]) + + def test_no_override_file_matches_nonexistent_override_path(self): + """AC1 的核心回归：不传覆盖文件时价格表就是纯内置表， + 与"指向一个不存在的路径"的输出完全一致（忽略 generated_at）。""" + with tempfile.TemporaryDirectory() as td: + root = Path(td) + unset, _ = self._run_main(root, "unset.json") + os.environ[OVERRIDES_ENV] = str(root / "nope.json") + emitter.clear_price_cache() + missing, _ = self._run_main(root, "missing.json") + unset.pop("generated_at") + missing.pop("generated_at") + self.assertEqual(unset, missing) + self.assertEqual(unset["unpricedModels"], [{"name": "zz-unpriced-a", "usageEvents": 2}]) + + +if __name__ == "__main__": + unittest.main() diff --git a/.cursor/skills/agent-observability/tests/test_runtime_flow.py b/.cursor/skills/agent-observability/tests/test_runtime_flow.py new file mode 100644 index 0000000..7bd143b --- /dev/null +++ b/.cursor/skills/agent-observability/tests/test_runtime_flow.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path +from unittest import mock +import sys + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "scripts" +if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) + +from core import runtime # type: ignore + + +class RuntimeFlowTests(unittest.TestCase): + def test_resolve_active_agent_prefers_inbox_inferred_agent_when_available(self): + data = { + "tool_name": "Task", + "tool_input": {"subagent_name": "developer"}, + } + with tempfile.TemporaryDirectory() as td: + state_path = Path(td) / ".state.json" + with mock.patch.object(runtime.agent_identity, "merge_agent_identity_from_inbox", return_value=("qa", "developer")) as merge_identity: + active, dispatched = runtime.agent_identity.resolve_active_agent_for_event( + state_path=state_path, + sid="s-inbox", + cwd=td, + data=data, + ) + + self.assertEqual(active, "qa") + self.assertEqual(dispatched, "developer") + merge_identity.assert_called_once() + + def test_handle_post_routes_through_expected_collaborators(self): + data = { + "session_id": "s-runtime", + "cwd": "/tmp/demo", + "tool_name": "Read", + "transcript_path": "/tmp/demo.jsonl", + } + with tempfile.TemporaryDirectory() as td: + with mock.patch.object(runtime, "build_runtime_paths") as mock_paths, \ + mock.patch.object(runtime.collector, "record_post", return_value=12), \ + mock.patch.object(runtime.collector, "load_cached_inventory", return_value=({}, {})), \ + mock.patch.object(runtime.scanner, "scan_skills_and_rules", return_value=({}, {})), \ + mock.patch.object(runtime.collector, "cache_inventory"), \ + mock.patch.object(runtime.collector, "record_tool_usage", return_value=([], [])), \ + mock.patch.object(runtime.agent_identity, "resolve_active_agent_for_event", return_value=("main", None)), \ + mock.patch.object(runtime, "current_turn_id", return_value="turn-1"), \ + mock.patch.object(runtime.collector, "extract_tool_call_id", return_value=None), \ + mock.patch.object(runtime, "flush_pending_tool_events"), \ + mock.patch.object(runtime.collector, "find_current_tool_context", return_value=None), \ + mock.patch.object(runtime.collector, "related_transcript_paths", return_value=["/tmp/demo.jsonl"]), \ + mock.patch.object(runtime, "emit_transcript_events", return_value=[]), \ + mock.patch.object(runtime.collector, "find_fallback_usage_event", return_value=None), \ + mock.patch.object(runtime.st, "append_pending_tool_emit"), \ + mock.patch.object(runtime.agentlens, "emit_post_step"): + mock_paths.return_value = runtime.RuntimePaths( + base_dir=Path(td), + state_path=Path(td) / ".state.json", + pending_path=Path(td) / ".pending.json", + log_path=Path(td) / "metrics.ndjson", + ) + runtime.handle_post(data) + + def test_main_dispatches_supported_phase_to_handler(self): + with mock.patch.object(runtime, "handle_post") as handle_post, \ + mock.patch.object(runtime.collector, "read_stdin_json", return_value={"session_id": "s1"}): + rc = runtime.main(["post"]) + self.assertEqual(rc, 0) + handle_post.assert_called_once() diff --git a/.gitignore b/.gitignore index 363f232..00c9449 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,14 @@ artifacts/ *.db *.sqlite __pycache__/ + +# agent-observability 运行时产物（本地日志、看板数据快照）：不入库，四个 host 副本都要排除 +.codebuddy/skills/agent-observability/logs/* +.codex/skills/agent-observability/logs/* +.cursor/skills/agent-observability/logs/* +.claude/skills/agent-observability/logs/* +!**/skills/agent-observability/logs/.gitkeep +**/skills/agent-observability/scripts/dashboard/dashboard-data.json *.py[cod] build/ dist/ diff --git a/scripts/build-classic-hosts.py b/scripts/build-classic-hosts.py index 52613c5..a8cfcbb 100644 --- a/scripts/build-classic-hosts.py +++ b/scripts/build-classic-hosts.py @@ -206,9 +206,15 @@ def write_host(host: str) -> None: def check_host(host: str) -> list: target = ROOT / f".{host}" wanted = expected(host) + # __pycache__/*.pyc 是运行 Python 工具（比如 validate.sh 里的 py_compile）时的 + # 正常副作用，不是宿主包内容本身；add_tree() 生成期望清单时已经排除了同样的 + # 东西（见上面 134 行），这里扫描"实际有什么"必须用一样的排除规则，否则任何 + # 在 .claude/.cursor 下跑过一次 Python 语法检查就会把这些缓存文件误判成 + # "unmanaged extra"。 actual = { str(path.relative_to(target)): path.read_bytes() - for path in target.rglob("*") if path.is_file() + for path in target.rglob("*") + if path.is_file() and "__pycache__" not in path.parts and path.suffix != ".pyc" } if target.is_dir() else {} errors = [] for name in sorted(set(wanted) | set(actual)):