diff --git a/codewiki/src/be/agent_orchestrator.py b/codewiki/src/be/agent_orchestrator.py index d538e678..5a2113b3 100644 --- a/codewiki/src/be/agent_orchestrator.py +++ b/codewiki/src/be/agent_orchestrator.py @@ -1,3 +1,13 @@ +"""Agent orchestration for documentation generation. + +This module defines AgentOrchestrator, the component responsible for +creating and running pydantic_ai agents that generate documentation for +modules discovered in a repository. It selects agent configurations based +on module complexity, wires up the required tools and dependencies, and +drives the per-module documentation generation pipeline (loading/saving the +module tree, invoking the agent, and persisting generated docs). +""" + from pydantic_ai import Agent from pydantic_ai.usage import UsageLimits # import logfire @@ -61,9 +71,6 @@ class AgentOrchestrator: """Orchestrates the AI agents for documentation generation.""" def __init__(self, config: Config): - import logging - logger = logging.getLogger(__name__) - self.config = config self.fallback_models = create_fallback_models(config) self.custom_instructions = config.get_prompt_addition() if config else None @@ -207,4 +214,4 @@ async def process_module(self, module_name: str, components: Dict[str, Node], except Exception as e: logger.error(f"❌ Error processing module {module_name}: {str(e)}") logger.error(f" └─ Traceback: {traceback.format_exc()}") - raise \ No newline at end of file + raise diff --git a/codewiki/src/be/agent_tools/deps.py b/codewiki/src/be/agent_tools/deps.py index 6f2c469a..f3eef7e1 100644 --- a/codewiki/src/be/agent_tools/deps.py +++ b/codewiki/src/be/agent_tools/deps.py @@ -1,4 +1,10 @@ +"""Dependency-injection context for the CodeWiki agent tools pipeline. + +Defines the CodeWikiDeps dataclass, which carries paths, registry data, +component metadata, and configuration through the agent tools pipeline. +""" from dataclasses import dataclass +from typing import Any from codewiki.src.be.dependency_analyzer.models.core import Node from codewiki.src.config import Config @@ -10,8 +16,8 @@ class CodeWikiDeps: components: dict[str, Node] path_to_current_module: list[str] current_module_name: str - module_tree: dict[str, any] + module_tree: dict[str, Any] max_depth: int current_depth: int config: Config # LLM configuration - custom_instructions: str = None \ No newline at end of file + custom_instructions: str = None diff --git a/codewiki/src/be/agent_tools/generate_sub_module_documentations.py b/codewiki/src/be/agent_tools/generate_sub_module_documentations.py index f0c7a938..ec68adc7 100644 --- a/codewiki/src/be/agent_tools/generate_sub_module_documentations.py +++ b/codewiki/src/be/agent_tools/generate_sub_module_documentations.py @@ -1,3 +1,21 @@ +"""Sub-module documentation generation pipeline step. + +This module implements the recursive agent-dispatch tool used by CodeWiki to +split a module into smaller sub-modules and generate documentation for each +one. It is responsible for: + +- Normalizing the component identifiers returned by the LLM (which may be + either FQDN strings or integer IDs from the ID-based clustering system) + into canonical FQDNs that exist in ``deps.components``. +- Updating the in-memory module tree with the newly created sub-modules. +- Spawning nested ``pydantic_ai`` agents (leaf or non-leaf, depending on + module complexity and depth) to recursively generate documentation for + each sub-module. + +It is exposed to the top-level documentation agent as +``generate_sub_module_documentation_tool``. +""" + from pydantic_ai import RunContext, Tool, Agent from pydantic_ai.usage import UsageLimits @@ -7,7 +25,7 @@ from codewiki.src.be.llm_services import create_fallback_models from codewiki.src.be.prompt_template import SYSTEM_PROMPT, LEAF_SYSTEM_PROMPT, format_user_prompt, format_system_prompt, format_leaf_system_prompt from codewiki.src.be.utils import is_complex_module, count_tokens -from codewiki.src.be.cluster_modules import format_potential_core_components +from codewiki.src.be.cluster_modules import format_potential_core_components, normalize_component_ids_by_lookup import logging logger = logging.getLogger(__name__) @@ -40,45 +58,9 @@ async def generate_sub_module_documentation( # This returns (str1, str2, id_to_fqdn, id_descriptions) but we only need id_to_fqdn _, _, id_to_fqdn, _ = format_potential_core_components(all_component_ids, deps.components) - normalized_specs = {} - total_normalized = 0 - total_failed = 0 - - for sub_module_name, component_ids in sub_module_specs.items(): - normalized_ids = [] - for comp_id in component_ids: - # Try exact FQDN match first (component_ids might already be FQDNs) - if comp_id in deps.components: - normalized_ids.append(comp_id) - # Try converting integer ID to FQDN (ID-based system) - else: - try: - # LLM should return integer IDs - idx = int(comp_id) - if idx in id_to_fqdn: - fqdn = id_to_fqdn[idx] - normalized_ids.append(fqdn) - total_normalized += 1 - logger.debug(f" ✅ Normalized ID {idx} → '{fqdn}'") - else: - logger.warning( - f" ⚠️ Failed to normalize ID {idx} in sub-module '{sub_module_name}'\n" - f" ├─ ID out of range (valid: 0-{len(id_to_fqdn)-1})\n" - f" └─ LLM returned invalid integer ID" - ) - total_failed += 1 - except (ValueError, TypeError): - # comp_id is not an integer - likely a class name (LLM ignored instructions) - similar_fqdns = [fqdn for fqdn in deps.components.keys() if str(comp_id).lower() in fqdn.lower()][:5] - logger.warning( - f" ⚠️ Failed to normalize '{comp_id}' in sub-module '{sub_module_name}'\n" - f" ├─ Not an integer ID (type: {type(comp_id).__name__})\n" - f" ├─ LLM returned class name instead of integer ID\n" - f" └─ FQDNs containing '{comp_id}': {similar_fqdns if similar_fqdns else 'None found'}" - ) - total_failed += 1 - - normalized_specs[sub_module_name] = normalized_ids + normalized_specs, total_normalized, total_failed = normalize_component_ids_by_lookup( + sub_module_specs, deps.components, id_to_fqdn + ) if total_normalized > 0: logger.info(f" ✅ Normalized {total_normalized} integer IDs to FQDNs") @@ -172,28 +154,22 @@ async def generate_sub_module_documentation( description="""Generate detailed documentation for sub-modules by grouping related components. CRITICAL FORMAT REQUIREMENTS: -- Use the EXACT component identifiers as shown in the section +- Use the EXACT integer component IDs as shown in the section - DO NOT extract just class names (e.g., "AuthService", "ApiApplicationConfig") -- Use the COMPLETE identifiers like: "main-repo.src/services/auth.py::AuthService" +- DO NOT invent full FQDN strings; use the integer IDs assigned to each component Example CORRECT format: { - "Authentication": [ - "main-repo.src/services/auth.py::AuthService", - "main-repo.src/services/auth.py::LoginController" - ], - "Configuration": [ - "main-repo.src/config/api.py::ApiApplicationConfig", - "main-repo.src/config/security.py::SecurityConfig" - ] + "Authentication": [0, 1], + "Configuration": [2, 3] } Example WRONG format (DO NOT USE): { "Authentication": ["AuthService", "LoginController"], # ❌ Class names only - "Configuration": ["ApiApplicationConfig"] # ❌ Missing full path + "Configuration": ["main-repo.src/config/api.py::ApiApplicationConfig"] # ❌ Full FQDN string instead of integer ID } -The component identifiers must match exactly what appears in .""", +The integer IDs must match exactly what appears in .""", takes_ctx=True -) \ No newline at end of file +) diff --git a/codewiki/src/be/dependency_analyzer/analysis/cloning.py b/codewiki/src/be/dependency_analyzer/analysis/cloning.py index 9bf4eedf..fe5f1c72 100644 --- a/codewiki/src/be/dependency_analyzer/analysis/cloning.py +++ b/codewiki/src/be/dependency_analyzer/analysis/cloning.py @@ -1,3 +1,13 @@ +"""Repository cloning and cleanup utilities. + +This module implements the repository acquisition step of the dependency +analysis pipeline: given a GitHub URL, it sanitizes and validates the URL, +clones the repository into a temporary directory for analysis, and safely +cleans up that directory afterwards (including handling Windows-specific +read-only file permission issues). Downstream analysis stages in the +dependency_analyzer package operate on the local clone produced here. +""" + import os import shutil import tempfile @@ -259,4 +269,4 @@ def parse_github_url(github_url: str) -> dict: "name": "unknown", "full_name": "unknown", "url": github_url, - } \ No newline at end of file + } diff --git a/codewiki/src/be/dependency_analyzer/analyzers/c.py b/codewiki/src/be/dependency_analyzer/analyzers/c.py index 9332a6f8..aa93bcc3 100644 --- a/codewiki/src/be/dependency_analyzer/analyzers/c.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/c.py @@ -1,3 +1,11 @@ +"""Tree-sitter based analyzer for C source files. + +This module parses C source code using tree-sitter to extract call-graph +nodes (functions, structs, and global variables) and the call/usage +relationships between them. It is part of the multi-language dependency +analysis pipeline, producing `Node` and `CallRelationship` objects that +feed into the broader dependency graph and clustering system. +""" import logging from typing import List, Optional, Tuple from pathlib import Path @@ -45,7 +53,7 @@ def _get_relative_path(self) -> str: def _get_component_id(self, name: str) -> str: module_path = self._get_module_path() - return f"{module_path}.{name}" if module_path else name + return f"{module_path}::{name}" if module_path else name def _analyze(self): language_capsule = tree_sitter_c.language() @@ -220,3 +228,4 @@ def _is_system_function(self, func_name: str) -> bool: def analyze_c_file(file_path: str, content: str, repo_path: str = None) -> Tuple[List[Node], List[CallRelationship]]: analyzer = TreeSitterCAnalyzer(file_path, content, repo_path) return analyzer.nodes, analyzer.call_relationships + diff --git a/codewiki/src/be/dependency_analyzer/analyzers/typescript.py b/codewiki/src/be/dependency_analyzer/analyzers/typescript.py index 68abc86d..818967e4 100644 --- a/codewiki/src/be/dependency_analyzer/analyzers/typescript.py +++ b/codewiki/src/be/dependency_analyzer/analyzers/typescript.py @@ -1,3 +1,13 @@ +"""TypeScript dependency analyzer built on tree-sitter. + +This module parses TypeScript/TSX source files using the `tree_sitter_typescript` +grammar to extract top-level declarations (functions, classes, interfaces, +type aliases, enums, variables, etc.) as `Node` objects and to infer call, +instantiation, member-access, type-usage, and inheritance relationships +between them as `CallRelationship` objects. It is invoked by the dependency +analysis pipeline for `.ts`/`.tsx` files to build the project's dependency +graph. +""" import logging import os import traceback @@ -199,6 +209,7 @@ def _get_parent_context(self, node) -> str: if node.parent.parent and node.parent.parent.type in ["module", "ambient_declaration"]: return "module_block" return "statement_block" + return "unknown" def _extract_function_entity(self, node, func_type: str, depth: int) -> dict: name_node = self._find_child_by_type(node, "identifier") if not name_node: @@ -979,4 +990,4 @@ def analyze_typescript_file_treesitter( return analyzer.nodes, analyzer.call_relationships except Exception as e: logger.error(f"Error in tree-sitter TS analysis for {file_path}: {e}", exc_info=True) - return [], [] \ No newline at end of file + return [], [] diff --git a/codewiki/src/be/dependency_analyzer/dependency_graphs_builder.py b/codewiki/src/be/dependency_analyzer/dependency_graphs_builder.py index c49612c8..96d7abe7 100644 --- a/codewiki/src/be/dependency_analyzer/dependency_graphs_builder.py +++ b/codewiki/src/be/dependency_analyzer/dependency_graphs_builder.py @@ -1,3 +1,10 @@ +"""Dependency graph construction for repository analysis. + +This module defines the DependencyGraphBuilder class, which orchestrates +parsing a repository's source files, building a dependency graph from the +extracted components, validating graph completeness, and filtering leaf +nodes to those relevant for downstream processing. +""" from typing import Dict, List, Any import os from codewiki.src.config import Config @@ -154,4 +161,4 @@ def build_dependency_graph(self) -> tuple[Dict[str, Any], List[str]]: logger.info(f" ├─ Skipped (wrong type): {skipped_type}") logger.info(f" └─ Skipped (not found): {skipped_not_found}") - return components, keep_leaf_nodes \ No newline at end of file + return components, keep_leaf_nodes diff --git a/codewiki/src/be/dependency_analyzer/models/core.py b/codewiki/src/be/dependency_analyzer/models/core.py index 78f768fa..3902c374 100644 --- a/codewiki/src/be/dependency_analyzer/models/core.py +++ b/codewiki/src/be/dependency_analyzer/models/core.py @@ -1,3 +1,8 @@ +"""Core pydantic data models for the dependency analyzer. + +Defines the Node, CallRelationship, and Repository models that form the +core data contract used throughout the dependency analysis pipeline. +""" from pydantic import BaseModel from typing import List, Optional, Dict, Any, Set from datetime import datetime @@ -66,3 +71,4 @@ class Repository(BaseModel): clone_path: str analysis_id: str + diff --git a/codewiki/src/be/dependency_analyzer/utils/security.py b/codewiki/src/be/dependency_analyzer/utils/security.py index 724421e6..cabd76ae 100644 --- a/codewiki/src/be/dependency_analyzer/utils/security.py +++ b/codewiki/src/be/dependency_analyzer/utils/security.py @@ -1,3 +1,11 @@ +"""Security utilities for safe file access within a repository root. + +This module implements path-traversal and symlink protections used when +reading files from a repository. It ensures that file access is confined to +a given base directory and that symlinks are not followed, mitigating +directory-traversal and symlink-escape attacks during dependency analysis. +""" + from pathlib import Path import os @@ -31,3 +39,4 @@ def safe_open_text(base_dir: Path, target: Path, encoding="utf-8"): os.close(fd) except OSError: pass + diff --git a/codewiki/src/config.py b/codewiki/src/config.py index 1e454424..edf41823 100644 --- a/codewiki/src/config.py +++ b/codewiki/src/config.py @@ -1,3 +1,18 @@ +"""Configuration module for CodeWiki. + +This module defines the central `Config` dataclass used throughout the +CodeWiki documentation-generation pipeline. It encapsulates repository +paths, output directories, LLM provider settings (models, API keys, base +URLs, temperatures, and token limits), and agent instruction customization +(include/exclude patterns, focus modules, doc type, custom instructions). + +It also provides constructors for building a `Config` instance from CLI +arguments (`from_args`), from explicit CLI parameters (`from_cli`), and +from a `ConfigManager` (`from_config_manager`), along with helpers for +multi-path source validation and prompt-addition generation used by the +downstream documentation generation stages. +""" + from dataclasses import dataclass, field from typing import Optional, List, Dict, Any import argparse @@ -687,4 +702,4 @@ def from_config_manager( agent_instructions=config_obj.agent_instructions.to_dict() if config_obj.agent_instructions else None, diagrams_dir=None, additional_source_paths=additional_paths - ) \ No newline at end of file + ) diff --git a/codewiki/src/utils.py b/codewiki/src/utils.py index 718e755c..19a0d50a 100644 --- a/codewiki/src/utils.py +++ b/codewiki/src/utils.py @@ -1,3 +1,5 @@ +"""File I/O utility helpers used across the CodeWiki backend and web app.""" + import os import json from typing import Any, Optional, Dict @@ -45,3 +47,4 @@ def load_text(filepath: str) -> str: return f.read() file_manager = FileManager() +