diff --git a/.code-reducer.yaml b/.code-reducer.yaml index 71c8965..eb25cc8 100644 --- a/.code-reducer.yaml +++ b/.code-reducer.yaml @@ -1,4 +1,4 @@ -model_id: gemma4:26b-a4b-it-qat +model_id: ornith:9b ollama_base_url: http://localhost:11434 ollama_num_ctx: 20000 docs_dir: wiki @@ -47,6 +47,9 @@ ignore: - go.mod - screenshots - examples - - docs + - gh-pages - LICENSE - AGENTS.md + - mkdocs.yml + - sonar-project.properties + diff --git a/cmd/cmd_test.go b/cmd/cmd_test.go index 8540042..350dc5b 100644 --- a/cmd/cmd_test.go +++ b/cmd/cmd_test.go @@ -97,13 +97,10 @@ func TestPromptStringList(t *testing.T) { input := "a, b, c\n" reader := bufio.NewReader(bytes.NewBufferString(input)) - result, modified, err := promptStringList(reader, "Prompt", []string{"default"}) + result, err := promptStringList(reader, "Prompt", []string{"default"}) if err != nil { t.Errorf("Unexpected error: %v", err) } - if !modified { - t.Errorf("Expected modified to be true") - } if len(result) != 3 || result[0] != "a" || result[1] != "b" || result[2] != "c" { t.Errorf("Unexpected result: %v", result) } @@ -111,13 +108,10 @@ func TestPromptStringList(t *testing.T) { // Test empty input falls back input = "\n" reader = bufio.NewReader(bytes.NewBufferString(input)) - result, modified, err = promptStringList(reader, "Prompt", []string{"default"}) + result, err = promptStringList(reader, "Prompt", []string{"default"}) if err != nil { t.Errorf("Unexpected error: %v", err) } - if modified { - t.Errorf("Expected modified to be false") - } if len(result) != 1 || result[0] != "default" { t.Errorf("Unexpected result: %v", result) } @@ -166,7 +160,7 @@ func TestPromptIgnores(t *testing.T) { func TestLoadInitialSetupConfig(t *testing.T) { tmp := t.TempDir() cfg := loadInitialSetupConfig(tmp) - if cfg.ModelID == "" || cfg.OllamaNumCtx <= 0 { + if cfg.ModelID == "" || cfg.OllamaNumCtx <= 0 || cfg.QuickstartPrompt == "" { t.Fatalf("expected non-empty defaults, got %+v", cfg) } @@ -178,8 +172,8 @@ func TestLoadInitialSetupConfig(t *testing.T) { t.Fatalf("failed to save config: %v", err) } loaded := loadInitialSetupConfig(tmp) - if loaded.ModelID != "custom-model" || loaded.OllamaNumCtx != 8192 { - t.Fatalf("expected custom values, got %+v", loaded) + if loaded.ModelID != "custom-model" || loaded.OllamaNumCtx != 8192 || loaded.QuickstartPrompt != config.DefaultQuickstartPrompt { + t.Fatalf("expected custom values and default prompts, got %+v", loaded) } } @@ -199,4 +193,7 @@ func TestRunSetupFlow(t *testing.T) { if cfg.ModelID != config.OllamaDefaultModelID { t.Errorf("expected default model ID, got %s", cfg.ModelID) } + if cfg.QuickstartPrompt != config.DefaultQuickstartPrompt { + t.Errorf("expected default quickstart prompt, got %s", cfg.QuickstartPrompt) + } } diff --git a/cmd/init.go b/cmd/init.go index c10204b..8532199 100644 --- a/cmd/init.go +++ b/cmd/init.go @@ -10,10 +10,7 @@ var initCmd = &cobra.Command{ Short: "Initialize project documentation", Long: `Scan the repository and build the initial set of wiki markdown pages.`, RunE: func(cmd *cobra.Command, args []string) error { - if err := executeCommand(engine.ModeInit); err != nil { - return err - } - return nil + return executeCommand(engine.ModeInit) }, } diff --git a/cmd/root.go b/cmd/root.go index fafca78..e7e04f5 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -34,7 +34,7 @@ func init() { RootCmd.PersistentFlags().StringVar(&numCtxFlag, "num-ctx", "", "Specify Ollama context window size") } -func executeCommand(mode engine.Mode) error { +func executeCommand(mode engine.RunMode) error { repoRoot, err := os.Getwd() if err != nil { return fmt.Errorf("failed to get current working directory: %w", err) @@ -61,53 +61,34 @@ func executeCommand(mode engine.Mode) error { } func checkAndRunSetup(repoRoot string) error { - needsSetup := !config.ConfigExists(repoRoot) + if config.ConfigExists(repoRoot) { + return nil + } isTTY := isatty.IsTerminal(os.Stdin.Fd()) || isatty.IsCygwinTerminal(os.Stdin.Fd()) - - if needsSetup { - if !isTTY { - return fmt.Errorf("configuration file %s does not exist in the current directory. Please run 'code-reducer setup' to configure the application", config.ConfigFileName) - } - - if err := RunSetupFlow(repoRoot); err != nil { - return err - } + if !isTTY { + return fmt.Errorf("configuration file %s does not exist in the current directory. Please run 'code-reducer setup' to configure the application", config.ConfigFileName) } - return nil + return RunSetupFlow(repoRoot) } -func checkInitStatus(repoRoot, docsDir string, mode engine.Mode) error { +func checkInitStatus(repoRoot, docsDir string, mode engine.RunMode) error { hasInit := engine.IsInitialized(repoRoot, docsDir) - if mode == engine.ModeInit { - if hasInit { - return fmt.Errorf("the project has already been initialized. Please use 'code-reducer update' to refresh documentation") - } - } else if mode == engine.ModeUpdate { - if !hasInit { - return fmt.Errorf("the project has not been initialized yet. Please run 'code-reducer init' first") - } + if mode == engine.ModeInit && hasInit { + return fmt.Errorf("the project has already been initialized. Please use 'code-reducer update' to refresh documentation") + } + if mode == engine.ModeUpdate && !hasInit { + return fmt.Errorf("the project has not been initialized yet. Please run 'code-reducer init' first") } return nil } -func runEngine(repoRoot string, cfg *config.Config, mode engine.Mode) error { +func runEngine(repoRoot string, cfg *config.Config, mode engine.RunMode) error { ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() - runner := engine.NewRunner(cfg) - err := runner.Run(ctx, repoRoot, mode, func(ev engine.Event) { - if ev.Type == engine.EventStatus { - fmt.Println(ev.Message) - } else if ev.Type == engine.EventError { - fmt.Fprintf(os.Stderr, "Error: %s\n", ev.Message) - } else { - fmt.Println(ev.Message) - } - }) - - if err != nil { + if err := engine.Run(ctx, repoRoot, cfg, mode, func(msg string) { fmt.Println(msg) }); err != nil { return fmt.Errorf("documentation run failed: %w", err) } diff --git a/cmd/setup.go b/cmd/setup.go index eb8a4db..61ab296 100644 --- a/cmd/setup.go +++ b/cmd/setup.go @@ -32,47 +32,10 @@ func init() { func loadInitialSetupConfig(repoRoot string) *config.Config { cfg, err := config.LoadConfig(repoRoot) - if err != nil || cfg == nil { - return &config.Config{ - ModelID: config.OllamaDefaultModelID, - OllamaBaseURL: config.OllamaDefaultBaseURL, - OllamaNumCtx: config.OllamaDefaultNumCtx, - DocsDir: config.DefaultDocsDir, - ExtractionSteps: config.DefaultExtractionSteps, - SystemPrompt: config.DefaultSystemPrompt, - ModuleSynthesisPrompt: config.DefaultModuleSynthesisPrompt, - ArchitecturePrompt: config.DefaultArchitecturePrompt, - FileFactConsolidationPrompt: config.DefaultFileFactConsolidationPrompt, - } - } - - if cfg.ModelID == "" { - cfg.ModelID = config.OllamaDefaultModelID - } - if cfg.OllamaBaseURL == "" { - cfg.OllamaBaseURL = config.OllamaDefaultBaseURL - } - if cfg.OllamaNumCtx <= 0 { - cfg.OllamaNumCtx = config.OllamaDefaultNumCtx - } - if cfg.DocsDir == "" { - cfg.DocsDir = config.DefaultDocsDir - } - if len(cfg.ExtractionSteps) == 0 { - cfg.ExtractionSteps = config.DefaultExtractionSteps - } - if cfg.SystemPrompt == "" { - cfg.SystemPrompt = config.DefaultSystemPrompt - } - if cfg.ModuleSynthesisPrompt == "" { - cfg.ModuleSynthesisPrompt = config.DefaultModuleSynthesisPrompt - } - if cfg.ArchitecturePrompt == "" { - cfg.ArchitecturePrompt = config.DefaultArchitecturePrompt - } - if cfg.FileFactConsolidationPrompt == "" { - cfg.FileFactConsolidationPrompt = config.DefaultFileFactConsolidationPrompt + if err != nil { + cfg = &config.Config{} } + config.ApplyDefaults(cfg) return cfg } @@ -88,20 +51,11 @@ func promptContextSize(reader *bufio.Reader, defaultVal int) (int, error) { } func promptIgnores(reader *bufio.Reader, defaultIgnores []string) ([]string, error) { - userInputIgnores, ignoresModified, err := promptStringList(reader, "Enter directories, files, or patterns to ignore (comma-separated)", defaultIgnores) + ignores, err := promptStringList(reader, "Enter directories, files, or patterns to ignore (comma-separated)", defaultIgnores) if err != nil { return nil, fmt.Errorf("error reading ignores: %w", err) } - if !ignoresModified { - if defaultIgnores != nil { - return defaultIgnores, nil - } - return []string{}, nil - } - if len(userInputIgnores) > 0 { - return userInputIgnores, nil - } - return []string{}, nil + return ignores, nil } // RunSetupFlow guides the user through setting up the configuration file. @@ -139,64 +93,37 @@ func runSetupFlowWithReader(reader *bufio.Reader, repoRoot string) error { return fmt.Errorf("error reading docs dir: %w", err) } - newCfg := &config.Config{ - ModelID: modelInput, - OllamaBaseURL: urlInput, - OllamaNumCtx: numCtx, - DocsDir: docsDirInput, - ExtractionSteps: current.ExtractionSteps, - Ignore: ignores, - SystemPrompt: current.SystemPrompt, - ModuleSynthesisPrompt: current.ModuleSynthesisPrompt, - ArchitecturePrompt: current.ArchitecturePrompt, - FileFactConsolidationPrompt: current.FileFactConsolidationPrompt, - } + current.ModelID = modelInput + current.OllamaBaseURL = urlInput + current.OllamaNumCtx = numCtx + current.DocsDir = docsDirInput + current.Ignore = ignores - err = config.SaveConfig(repoRoot, newCfg) - if err != nil { + if err := config.SaveConfig(repoRoot, current); err != nil { return fmt.Errorf("error saving configuration: %w", err) } fmt.Printf("Configuration successfully saved to local %s file.\n", config.ConfigFileName) return nil } -func promptStringList(reader *bufio.Reader, promptMsg string, existingList []string) ([]string, bool, error) { - var result []string - existingStr := "" - if len(existingList) > 0 { - existingStr = strings.Join(existingList, ", ") - } - - if existingStr == "" { - fmt.Printf("%s: ", promptMsg) - } else { - fmt.Printf("%s [%s]: ", promptMsg, existingStr) - } - - input, err := reader.ReadString('\n') +func promptStringList(reader *bufio.Reader, promptMsg string, existingList []string) ([]string, error) { + input, err := promptString(reader, promptMsg, strings.Join(existingList, ", ")) if err != nil { - return nil, false, err - } - - input = strings.TrimSpace(input) - if input == "" { - return existingList, false, nil + return nil, err } lowerInput := strings.ToLower(input) - if lowerInput == "clear" || lowerInput == "none" { - return []string{}, true, nil + if input == "" || lowerInput == "clear" || lowerInput == "none" { + return []string{}, nil } - parts := strings.Split(input, ",") - for _, part := range parts { - trimmed := strings.TrimSpace(part) - if trimmed != "" { + var result []string + for _, part := range strings.Split(input, ",") { + if trimmed := strings.TrimSpace(part); trimmed != "" { result = append(result, trimmed) } } - - return result, true, nil + return result, nil } func promptString(reader *bufio.Reader, promptMsg, existingVal string) (string, error) { diff --git a/cmd/update.go b/cmd/update.go index 199bc31..2f85328 100644 --- a/cmd/update.go +++ b/cmd/update.go @@ -7,13 +7,10 @@ import ( var updateCmd = &cobra.Command{ Use: "update", - Short: "Update project documentation incrementally", + Short: "Incrementally update documentation for modified files using cached SHA-256 hashes.", Long: `Scan changed files since the last documented commit and update the wiki pages.`, RunE: func(cmd *cobra.Command, args []string) error { - if err := executeCommand(engine.ModeUpdate); err != nil { - return err - } - return nil + return executeCommand(engine.ModeUpdate) }, } diff --git a/internal/config/config.go b/internal/config/config.go index 24f6a14..5df675d 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -21,17 +21,33 @@ const ( configFilePerm = 0600 // DefaultSystemPrompt is the default system instructions for the LLM. - DefaultSystemPrompt = "You are Code-Reducer, an expert technical writer and code analyzer. Your job is to strictly follow instructions. You do not yap, you do not write filler.\n" + - "DEFENSIVE RULES: 1. Do NOT use absolute terms ('always', 'never', 'zero') unless explicitly proven. 2. Do NOT guess downstream consequences or invent unhandled paths. If an error is swallowed, just say it is swallowed. 3. Do NOT name standard library packages unless explicitly stated in the source text. 4. Only report facts you are 100% sure about.\n" + DefaultSystemPrompt = "You are Code-Reducer, a technical code analyzer and documentation writer. Follow instructions strictly with maximum technical conciseness.\n" + + "Rules:\n" + + "1. Extract and report ONLY verifiable facts explicitly present in the provided source code.\n" + + "2. Do NOT invent behaviors, speculate on unshown code, or guess downstream consequences.\n" + + "3. Do NOT copy source code blocks or write conversational filler.\n" // DefaultModuleSynthesisPrompt is the default prompt for folder summaries. - DefaultModuleSynthesisPrompt = "Task: Write a technical documentation page for a code module based on the provided list of its internal components.\nRule 1: Group related functions and classes under appropriate Markdown headings.\nRule 2: Explain the responsibility of the module and the data flow.\nRule 3: Keep it highly technical and dense." + DefaultModuleSynthesisPrompt = "Task: Write a technical documentation page for a code module based on the provided list of its internal components.\n" + + "Rule 1: Group related functions and types under appropriate Markdown headings.\n" + + "Rule 2: Explain the responsibility of the module and its internal data flow.\n" + + "Rule 3: Keep it highly technical, concise, and free of source code blocks." - // DefaultArchitecturePrompt is the default prompt for architecture synthesis. - DefaultArchitecturePrompt = "Task: Write a global architecture or quickstart document based on the module summaries.\nRule 1: Explain the system boundaries and how the modules interact.\nRule 2: Provide a dense, developer-friendly overview." + // DefaultArchitecturePrompt is the default prompt for global architecture synthesis. + DefaultArchitecturePrompt = "Task: Write the global architecture document (architecture.md) based on the module summaries.\n" + + "Rule 1: Focus exclusively on the global architecture vision, system boundaries, end-to-end data flow, and module interactions.\n" + + "Rule 2: Keep it concise, technical, and developer-friendly." + + // DefaultQuickstartPrompt is the default prompt for developer quickstart synthesis. + DefaultQuickstartPrompt = "Task: Write the developer quickstart guide (quickstart.md) based on the module summaries.\n" + + "Rule 1: Focus exclusively on prerequisites, build/installation steps, available CLI commands and flags, configuration (.code-reducer.yaml and environment variables), and package structure.\n" + + "Rule 2: Keep it concise, actionable, and strictly factual." // DefaultFileFactConsolidationPrompt is the default prompt for consolidation. - DefaultFileFactConsolidationPrompt = "You are a specialized code documentation assistant.\nConsolidate, deduplicate and merge the following facts extracted from different chunks of the same file into a single, cohesive summary." + DefaultFileFactConsolidationPrompt = "Task: Consolidate and deduplicate facts extracted from different chunks of the same file into a single, concise summary.\n" + + "Rule 1: Merge duplicate items and preserve only unique, verifiable facts.\n" + + "Rule 2: Keep bullet points to 1 line each; do NOT copy source code blocks or add long explanations.\n" + + "Rule 3: If all inputs are 'None', output solely 'None'." ) // ExtractionStep represents a single fact-extraction phase for files. @@ -49,6 +65,7 @@ type Config struct { SystemPrompt string `yaml:"system_prompt"` ModuleSynthesisPrompt string `yaml:"module_synthesis_prompt"` ArchitecturePrompt string `yaml:"architecture_prompt"` + QuickstartPrompt string `yaml:"quickstart_prompt"` FileFactConsolidationPrompt string `yaml:"file_fact_consolidation_prompt"` ExtractionSteps []ExtractionStep `yaml:"extraction_steps"` Ignore []string `yaml:"ignore"` @@ -58,18 +75,18 @@ type Config struct { var DefaultExtractionSteps = []ExtractionStep{ { Name: "API_SIGNATURES", - Prompt: "Task: Extract the public API surface.\nOutput: A strict Markdown list of all exported or public elements (classes, functions, methods, types). Include parameters and return types. Do not explain internal execution logic.", + Prompt: "Task: Extract the public API surface.\nOutput: Markdown bullet list of exported/public types, functions, and methods, strictly 1 line per signature with parameters and return types. Do NOT explain internal logic or copy code blocks. If none exist, output solely 'None'.", }, { Name: "BUSINESS_LOGIC", - Prompt: "Task: Extract the core purpose and domain rules.\nOutput: Explain the primary domain problem this code solves. List the high-level algorithm steps. Ignore syntax, standard library usage, and basic implementation details.", + Prompt: "Task: Extract the core purpose and domain rules.\nOutput: Maximum 5 concise bullet points describing the primary domain problem and high-level logic. Ignore syntax and boilerplate. Do NOT copy code blocks. If none exist, output solely 'None'.", }, { Name: "STATE_AND_CONCURRENCY", - Prompt: "Task: Identify mutable state and thread safety.\nOutput: List global variables, shared states, or class-level properties that are modified. Identify synchronization mechanisms (locks, mutexes, async/await, atomic types). If entirely stateless, output exactly: 'No mutable state'.", + Prompt: "Task: Identify mutable state and thread safety.\nOutput: Concise bullet list (1 line per item) of modified package/global state, struct fields, or synchronization mechanisms (locks, mutexes, channels, atomics). Do NOT copy code blocks. If stateless, output solely 'None'.", }, { Name: "ERRORS_AND_SIDE_EFFECTS", - Prompt: "Task: Analyze external I/O and error propagation.\nOutput: Detail interactions with external systems (network, disk, databases, APIs). Explain how errors are propagated (exceptions, error return codes, crash/panic). If no I/O exists, state 'No external side effects'.", + Prompt: "Task: Analyze external I/O and error propagation.\nOutput: Concise bullet list (1 line per item) of external system interactions (filesystem, network, subprocesses, environment) and error handling patterns. Do NOT copy code blocks. If none exist, output solely 'None'.", }, } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index fe44d79..a2b5b67 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -68,6 +68,9 @@ func TestResolveConfig(t *testing.T) { if resolved.ModelID != "flag-model" { t.Errorf("expected flag-model, got %s", resolved.ModelID) } + if resolved.QuickstartPrompt != DefaultQuickstartPrompt { + t.Errorf("expected default quickstart prompt, got %s", resolved.QuickstartPrompt) + } }) t.Run("fail fast on invalid num ctx env", func(t *testing.T) { diff --git a/internal/config/io.go b/internal/config/io.go index cfa188e..1eade6b 100644 --- a/internal/config/io.go +++ b/internal/config/io.go @@ -7,6 +7,8 @@ import ( "strings" "gopkg.in/yaml.v3" + + "github.com/arrase/code-reducer/internal/tools" ) // getConfigPath returns the absolute path to the configuration file. @@ -35,16 +37,16 @@ func LoadConfig(cwd string) (*Config, error) { func formatYAML(data []byte) string { yamlStr := string(data) - replacements := []string{ - "\nsystem_prompt:", "\n\nsystem_prompt:", - "\nmodule_synthesis_prompt:", "\n\nmodule_synthesis_prompt:", - "\narchitecture_prompt:", "\n\narchitecture_prompt:", - "\nfile_fact_consolidation_prompt:", "\n\nfile_fact_consolidation_prompt:", - "\nextraction_steps:", "\n\nextraction_steps:", - "\nignore:", "\n\nignore:", - } - for i := 0; i < len(replacements); i += 2 { - yamlStr = strings.ReplaceAll(yamlStr, replacements[i], replacements[i+1]) + for _, key := range []string{ + "system_prompt:", + "module_synthesis_prompt:", + "architecture_prompt:", + "quickstart_prompt:", + "file_fact_consolidation_prompt:", + "extraction_steps:", + "ignore:", + } { + yamlStr = strings.ReplaceAll(yamlStr, "\n"+key, "\n\n"+key) } return yamlStr } @@ -57,32 +59,5 @@ func SaveConfig(cwd string, cfg *Config) error { } yamlStr := formatYAML(data) - - tmpFile, err := os.CreateTemp(cwd, ConfigFileName+".tmp.*") - if err != nil { - return fmt.Errorf("failed to create temp file: %w", err) - } - tmpName := tmpFile.Name() - defer func() { - tmpFile.Close() - os.Remove(tmpName) - }() - - if _, err := tmpFile.Write([]byte(yamlStr)); err != nil { - return fmt.Errorf("failed to write config to temp file: %w", err) - } - if err := tmpFile.Sync(); err != nil { - return fmt.Errorf("failed to sync temp file: %w", err) - } - if err := tmpFile.Close(); err != nil { - return fmt.Errorf("failed to close temp file: %w", err) - } - if err := os.Chmod(tmpName, configFilePerm); err != nil { - return fmt.Errorf("failed to chmod temp file: %w", err) - } - - if err := os.Rename(tmpName, getConfigPath(cwd)); err != nil { - return fmt.Errorf("failed to rename temp config file: %w", err) - } - return nil + return tools.WriteFileAtomic(getConfigPath(cwd), []byte(yamlStr), configFilePerm) } diff --git a/internal/config/resolve.go b/internal/config/resolve.go index 1609c99..d97066a 100644 --- a/internal/config/resolve.go +++ b/internal/config/resolve.go @@ -4,7 +4,9 @@ import ( "errors" "fmt" "os" + "path/filepath" "strconv" + "strings" ) // deduplicate removes duplicates from a slice. @@ -20,11 +22,40 @@ func deduplicate(a []string) []string { return result } -func resolveString(yamlVal, defaultVal string) string { - if yamlVal != "" { - return yamlVal +// ApplyDefaults populates empty or zero fields in cfg with system default values. +func ApplyDefaults(cfg *Config) { + if cfg.ModelID == "" { + cfg.ModelID = OllamaDefaultModelID + } + if cfg.OllamaBaseURL == "" { + cfg.OllamaBaseURL = OllamaDefaultBaseURL + } + if cfg.OllamaNumCtx <= 0 { + cfg.OllamaNumCtx = OllamaDefaultNumCtx + } + cfg.DocsDir = strings.TrimSpace(cfg.DocsDir) + if cfg.DocsDir == "" { + cfg.DocsDir = DefaultDocsDir + } + cfg.DocsDir = filepath.ToSlash(filepath.Clean(cfg.DocsDir)) + if len(cfg.ExtractionSteps) == 0 { + cfg.ExtractionSteps = DefaultExtractionSteps + } + if cfg.SystemPrompt == "" { + cfg.SystemPrompt = DefaultSystemPrompt + } + if cfg.ModuleSynthesisPrompt == "" { + cfg.ModuleSynthesisPrompt = DefaultModuleSynthesisPrompt + } + if cfg.ArchitecturePrompt == "" { + cfg.ArchitecturePrompt = DefaultArchitecturePrompt + } + if cfg.QuickstartPrompt == "" { + cfg.QuickstartPrompt = DefaultQuickstartPrompt + } + if cfg.FileFactConsolidationPrompt == "" { + cfg.FileFactConsolidationPrompt = DefaultFileFactConsolidationPrompt } - return defaultVal } func loadBaseConfig(repoRoot string) (*Config, error) { @@ -39,10 +70,7 @@ func loadBaseConfig(repoRoot string) (*Config, error) { } func resolveModelID(cfgVal, flagVal string) string { - modelID := OllamaDefaultModelID - if cfgVal != "" { - modelID = cfgVal - } + modelID := cfgVal if envVal := os.Getenv(CodeReducerModelIDEnvKey); envVal != "" { modelID = envVal } @@ -53,32 +81,33 @@ func resolveModelID(cfgVal, flagVal string) string { } func resolveBaseURL(cfgVal string) string { - baseURL := OllamaDefaultBaseURL - if cfgVal != "" { - baseURL = cfgVal - } if envVal := os.Getenv(OllamaBaseURLEnvKey); envVal != "" { - baseURL = envVal + return envVal } - return baseURL + return cfgVal } -func resolveNumCtx(cfgVal int, flagVal string) (int, error) { - numCtx := OllamaDefaultNumCtx - if cfgVal > 0 { - numCtx = cfgVal +func parsePositiveInt(val, errMsg string) (int, error) { + n, err := strconv.Atoi(val) + if err != nil || n <= 0 { + return 0, fmt.Errorf("%s: %s", errMsg, val) } + return n, nil +} + +func resolveNumCtx(cfgVal int, flagVal string) (int, error) { + numCtx := cfgVal if envVal := os.Getenv(OllamaNumCtxEnvKey); envVal != "" { - n, err := strconv.Atoi(envVal) - if err != nil || n <= 0 { - return 0, fmt.Errorf("invalid value for %s: %s", OllamaNumCtxEnvKey, envVal) + n, err := parsePositiveInt(envVal, "invalid value for "+OllamaNumCtxEnvKey) + if err != nil { + return 0, err } numCtx = n } if flagVal != "" { - n, err := strconv.Atoi(flagVal) - if err != nil || n <= 0 { - return 0, fmt.Errorf("invalid value for num-ctx flag: %s", flagVal) + n, err := parsePositiveInt(flagVal, "invalid value for num-ctx flag") + if err != nil { + return 0, err } numCtx = n } @@ -92,27 +121,16 @@ func ResolveConfig(repoRoot, modelIDFlag, numCtxFlag string) (*Config, error) { if err != nil { return nil, err } + ApplyDefaults(cfg) numCtx, err := resolveNumCtx(cfg.OllamaNumCtx, numCtxFlag) if err != nil { return nil, err } - resolvedSteps := cfg.ExtractionSteps - if len(resolvedSteps) == 0 { - resolvedSteps = DefaultExtractionSteps - } - - return &Config{ - Ignore: deduplicate(cfg.Ignore), - ExtractionSteps: resolvedSteps, - ModelID: resolveModelID(cfg.ModelID, modelIDFlag), - OllamaBaseURL: resolveBaseURL(cfg.OllamaBaseURL), - OllamaNumCtx: numCtx, - DocsDir: resolveString(cfg.DocsDir, DefaultDocsDir), - SystemPrompt: resolveString(cfg.SystemPrompt, DefaultSystemPrompt), - ModuleSynthesisPrompt: resolveString(cfg.ModuleSynthesisPrompt, DefaultModuleSynthesisPrompt), - ArchitecturePrompt: resolveString(cfg.ArchitecturePrompt, DefaultArchitecturePrompt), - FileFactConsolidationPrompt: resolveString(cfg.FileFactConsolidationPrompt, DefaultFileFactConsolidationPrompt), - }, nil + cfg.Ignore = deduplicate(cfg.Ignore) + cfg.ModelID = resolveModelID(cfg.ModelID, modelIDFlag) + cfg.OllamaBaseURL = resolveBaseURL(cfg.OllamaBaseURL) + cfg.OllamaNumCtx = numCtx + return cfg, nil } diff --git a/internal/engine/cache.go b/internal/engine/cache.go index 8b1856f..f89d2d6 100644 --- a/internal/engine/cache.go +++ b/internal/engine/cache.go @@ -18,7 +18,10 @@ type FileCacheEntry struct { Facts string `json:"facts"` } -const currentCacheVersion = 1 +const ( + currentCacheVersion = 1 + metadataFileName = ".metadata.json" +) type MetadataCache struct { Version int `json:"version"` @@ -27,13 +30,23 @@ type MetadataCache struct { Modules map[string]string `json:"modules"` } -func computeStepsHash(steps []config.ExtractionStep) string { - data, err := json.Marshal(steps) - if err != nil { - return "" +func computeStepsHash(cfg *config.Config) string { + h := sha256.New() + writeField := func(s string) { + h.Write([]byte(s)) + h.Write([]byte{0}) } - h := sha256.Sum256(data) - return hex.EncodeToString(h[:]) + writeField(cfg.ModelID) + writeField(cfg.SystemPrompt) + writeField(cfg.ModuleSynthesisPrompt) + writeField(cfg.ArchitecturePrompt) + writeField(cfg.QuickstartPrompt) + writeField(cfg.FileFactConsolidationPrompt) + for _, step := range cfg.ExtractionSteps { + writeField(step.Name) + writeField(step.Prompt) + } + return hex.EncodeToString(h.Sum(nil)) } func newEmptyCache() *MetadataCache { @@ -51,11 +64,11 @@ func loadMetadataCache(repoRoot, docsDir string) (*MetadataCache, error) { if errors.Is(err, os.ErrNotExist) { return newEmptyCache(), nil } - return newEmptyCache(), fmt.Errorf("failed to read metadata cache: %w", err) + return nil, fmt.Errorf("failed to read metadata cache: %w", err) } var cache MetadataCache if err := json.Unmarshal(data, &cache); err != nil { - return newEmptyCache(), fmt.Errorf("failed to unmarshal metadata cache: %w", err) + return nil, fmt.Errorf("failed to unmarshal metadata cache: %w", err) } if cache.Version != currentCacheVersion { // Incompatible version: return a clean cache @@ -72,9 +85,7 @@ func loadMetadataCache(repoRoot, docsDir string) (*MetadataCache, error) { // IsInitialized checks if the metadata cache file exists. func IsInitialized(repoRoot, docsDir string) bool { - metadataPath := filepath.Join(docsDir, metadataFileName) - _, err := tools.ReadFileSafely(repoRoot, metadataPath) - return err == nil + return tools.FileExistsSafely(repoRoot, filepath.Join(docsDir, metadataFileName)) } func saveMetadataCache(repoRoot, docsDir string, cache *MetadataCache) error { diff --git a/internal/engine/chunking.go b/internal/engine/chunking.go index 0d68ee5..f44b2d6 100644 --- a/internal/engine/chunking.go +++ b/internal/engine/chunking.go @@ -9,12 +9,44 @@ import ( "github.com/arrase/code-reducer/internal/config" ) +const ( + defaultChunkOverlap = 800 + maxCharsMultiplier = 2 + partialBatchInstruction = "Summarize and consolidate the key technical facts from this partial batch without repeating boilerplate headers" +) + +// stripOuterMarkdownFence strips surrounding markdown code fences only if there are no intermediate fences. +func stripOuterMarkdownFence(text string) string { + trimmed := strings.TrimSpace(text) + lines := strings.Split(trimmed, "\n") + if len(lines) < 2 { + return trimmed + } + if !strings.HasPrefix(strings.TrimSpace(lines[0]), "```") || strings.TrimSpace(lines[len(lines)-1]) != "```" { + return trimmed + } + for _, line := range lines[1 : len(lines)-1] { + if strings.HasPrefix(strings.TrimSpace(line), "```") { + return trimmed + } + } + return strings.TrimSpace(strings.Join(lines[1:len(lines)-1], "\n")) +} + type reductionConfig struct { sysPrompt string buildPrompt func(batch []string) string logMsg func(batch []string) string errMsg string - logEvent LogEventFunc + onStatus StatusFunc +} + +func calculateFileLimit(numCtx int, sysPrompt string) int { + limit := numCtx*maxCharsMultiplier - utf8.RuneCountInString(sysPrompt) + if limit < defaultChunkOverlap*2 { + return defaultChunkOverlap * 2 + } + return limit } func reduceWithLLM( @@ -26,11 +58,12 @@ func reduceWithLLM( if len(items) == 0 { return "", nil } - maxChars := c.NumCtx() * maxCharsMultiplier + promptOverhead := cfg.sysPrompt + "\n" + cfg.buildPrompt(nil) + "\n" + partialBatchInstruction + maxChars := calculateFileLimit(c.NumCtx(), promptOverhead) return reduceItems(ctx, items, maxChars, func(batch []string) (string, error) { prompt := cfg.buildPrompt(batch) - cfg.logEvent(EventStatus, cfg.logMsg(batch)) - res, err := c.CallLLM(ctx, cfg.sysPrompt, []Message{{Role: "user", Content: prompt}}, false) + cfg.onStatus(cfg.logMsg(batch)) + res, err := c.CallLLM(ctx, cfg.sysPrompt, prompt) if err != nil { return "", fmt.Errorf("%s: %w", cfg.errMsg, err) } @@ -38,7 +71,7 @@ func reduceWithLLM( }) } -func reduceInChunks(ctx context.Context, c llmCaller, nodePath string, items []string, cfg *config.Config, logEvent LogEventFunc) (string, error) { +func reduceInChunks(ctx context.Context, c llmCaller, nodePath string, items []string, cfg *config.Config, onStatus StatusFunc) (string, error) { redCfg := reductionConfig{ sysPrompt: cfg.SystemPrompt + "\n" + cfg.ModuleSynthesisPrompt, buildPrompt: func(batch []string) string { @@ -48,14 +81,31 @@ func reduceInChunks(ctx context.Context, c llmCaller, nodePath string, items []s return fmt.Sprintf("➜ LLM Synthesizing chunk for %s (%d items)", nodePath, len(batch)) }, errMsg: "LLM error during synthesis", - logEvent: logEvent, + onStatus: onStatus, } return reduceWithLLM(ctx, c, items, redCfg) } -func reduceFileFacts(ctx context.Context, c llmCaller, filePath string, stepName string, items []string, cfg *config.Config, logEvent LogEventFunc) (string, error) { - if len(items) == 1 { - return items[0], nil +func isNoneFact(s string) bool { + cleaned := strings.TrimSpace(s) + cleaned = strings.TrimPrefix(cleaned, "- ") + cleaned = strings.TrimPrefix(cleaned, "* ") + cleaned = strings.TrimSuffix(strings.TrimSpace(cleaned), ".") + return cleaned == "" || strings.EqualFold(cleaned, "none") || strings.EqualFold(cleaned, "n/a") +} + +func reduceFileFacts(ctx context.Context, c llmCaller, filePath string, stepName string, items []string, cfg *config.Config, onStatus StatusFunc) (string, error) { + var validItems []string + for _, item := range items { + if !isNoneFact(item) { + validItems = append(validItems, item) + } + } + if len(validItems) == 0 { + return "", nil + } + if len(validItems) == 1 { + return validItems[0], nil } redCfg := reductionConfig{ sysPrompt: cfg.SystemPrompt + "\n" + cfg.FileFactConsolidationPrompt, @@ -66,9 +116,16 @@ func reduceFileFacts(ctx context.Context, c llmCaller, filePath string, stepName return fmt.Sprintf("➜ LLM Consolidating facts for %s (%d items)", filePath, len(batch)) }, errMsg: "LLM error during file fact consolidation", - logEvent: logEvent, + onStatus: onStatus, } - return reduceWithLLM(ctx, c, items, redCfg) + res, err := reduceWithLLM(ctx, c, validItems, redCfg) + if err != nil { + return "", err + } + if isNoneFact(res) { + return "", nil + } + return res, nil } func expandOversizedItems(items []string, maxChars int) ([]string, error) { @@ -139,7 +196,8 @@ func reduceItems(ctx context.Context, items []string, maxChars int, reduceFn fun if err := ctx.Err(); err != nil { return "", err } - chunkRes, err := reduceItems(ctx, batch, maxChars, reduceFn) + partialBatch := append([]string{partialBatchInstruction}, batch...) + chunkRes, err := reduceFn(partialBatch) if err != nil { return "", err } @@ -160,8 +218,8 @@ func chunkTextWithOverlap(text string, maxRunes int, overlapRunes int) ([]string if maxRunes <= 0 { return nil, fmt.Errorf("maxRunes must be greater than 0") } - if overlapRunes >= maxRunes { - return nil, fmt.Errorf("overlapRunes must be strictly less than maxRunes") + if overlapRunes < 0 || overlapRunes >= maxRunes { + return nil, fmt.Errorf("overlapRunes must be between 0 and maxRunes-1") } runes := []rune(text) @@ -170,16 +228,24 @@ func chunkTextWithOverlap(text string, maxRunes int, overlapRunes int) ([]string } var chunks []string - step := maxRunes - overlapRunes - for i := 0; i < len(runes); i += step { - end := i + maxRunes - if end > len(runes) { - end = len(runes) - } - chunks = append(chunks, string(runes[i:end])) - if end == len(runes) { + for start := 0; start < len(runes); { + end := start + maxRunes + if end >= len(runes) { + chunks = append(chunks, string(runes[start:])) break } + for j := end - 1; j >= start+(maxRunes*3)/4; j-- { + if runes[j] == '\n' { + end = j + 1 + break + } + } + chunks = append(chunks, string(runes[start:end])) + nextStart := end - overlapRunes + if nextStart <= start { + nextStart = end + } + start = nextStart } return chunks, nil } diff --git a/internal/engine/chunking_test.go b/internal/engine/chunking_test.go index 454b591..287601e 100644 --- a/internal/engine/chunking_test.go +++ b/internal/engine/chunking_test.go @@ -15,10 +15,12 @@ func TestChunkTextWithOverlap(t *testing.T) { }) t.Run("invalid overlapRunes", func(t *testing.T) { - _, err := chunkTextWithOverlap("test", 10, 10) - if err == nil { + if _, err := chunkTextWithOverlap("test", 10, 10); err == nil { t.Fatal("expected error for overlapRunes >= maxRunes") } + if _, err := chunkTextWithOverlap("test", 10, -1); err == nil { + t.Fatal("expected error for overlapRunes < 0") + } }) t.Run("text shorter than maxRunes", func(t *testing.T) { @@ -59,6 +61,48 @@ func TestChunkTextWithOverlap(t *testing.T) { t.Errorf("expected chunk 2 to be 'efg', got '%s'", chunks[2]) } }) + + t.Run("chunking splits at last newline in tail window", func(t *testing.T) { + text := "line1\nline2\nline3\n" + chunks, err := chunkTextWithOverlap(text, 14, 2) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(chunks) != 2 { + t.Fatalf("expected 2 chunks, got %d: %q", len(chunks), chunks) + } + if chunks[0] != "line1\nline2\n" { + t.Errorf("expected chunk 0 to end after newline 'line1\\nline2\\n', got %q", chunks[0]) + } + if chunks[1] != "2\nline3\n" { + t.Errorf("expected chunk 1 to preserve overlap '2\\nline3\\n', got %q", chunks[1]) + } + }) +} + +func TestStripOuterMarkdownFence(t *testing.T) { + t.Run("strips single outer markdown fence", func(t *testing.T) { + input := "```markdown\n# Title\nBody\n```" + got := stripOuterMarkdownFence(input) + if got != "# Title\nBody" { + t.Errorf("expected '# Title\\nBody', got %q", got) + } + }) + + t.Run("preserves document with multiple code blocks", func(t *testing.T) { + input := "```go\nfunc a() {}\n```\n\nSome text\n\n```go\nfunc b() {}\n```" + got := stripOuterMarkdownFence(input) + if got != input { + t.Errorf("expected multiple code blocks to remain intact, got %q", got) + } + }) + + t.Run("leaves plain text untouched", func(t *testing.T) { + input := "# Just Markdown\nSome content" + if got := stripOuterMarkdownFence(input); got != input { + t.Errorf("expected %q, got %q", input, got) + } + }) } func TestReduceItems(t *testing.T) { @@ -80,6 +124,9 @@ func TestReduceItems(t *testing.T) { t.Errorf("expected result to contain %s, got %s", item, res) } } + if !strings.Contains(res, "Summarize and consolidate the key technical facts from this partial batch without repeating boilerplate headers") { + t.Errorf("expected intermediate partial batch instruction in result, got %s", res) + } } func TestExpandOversizedItems(t *testing.T) { @@ -113,14 +160,15 @@ func TestCountRunes(t *testing.T) { func TestCalculateFileLimit(t *testing.T) { // Below floor - limit1 := calculateFileLimit(100) - if limit1 <= 0 { - t.Fatalf("expected positive limit, got %d", limit1) + limit1 := calculateFileLimit(100, "system prompt") + if limit1 != defaultChunkOverlap*2 { + t.Fatalf("expected floor %d, got %d", defaultChunkOverlap*2, limit1) } - // Above floor - limit2 := calculateFileLimit(8192) - if limit2 <= limit1 { - t.Fatalf("expected limit2 > limit1, got %d <= %d", limit2, limit1) + // Above floor, subtracting system prompt length + limit2 := calculateFileLimit(8192, "1234567890") + expected := 8192*maxCharsMultiplier - 10 + if limit2 != expected { + t.Fatalf("expected %d, got %d", expected, limit2) } } diff --git a/internal/engine/client.go b/internal/engine/client.go index 4872803..97391c6 100644 --- a/internal/engine/client.go +++ b/internal/engine/client.go @@ -8,6 +8,12 @@ import ( "io" "net/http" "strings" + "time" +) + +const ( + defaultHTTPTimeout = 10 * time.Minute + maxErrorBodyBytes = 1024 ) type Message struct { @@ -16,7 +22,7 @@ type Message struct { } type llmCaller interface { - CallLLM(ctx context.Context, systemPrompt string, messages []Message, jsonFormat bool) (string, error) + CallLLM(ctx context.Context, systemPrompt, userPrompt string) (string, error) NumCtx() int } @@ -24,14 +30,16 @@ type llmClient struct { modelID string baseURL string numCtx int + onStatus StatusFunc httpClient *http.Client } -func newLLMClient(modelID, baseURL string, numCtx int) *llmClient { +func newLLMClient(modelID, baseURL string, numCtx int, onStatus StatusFunc) *llmClient { return &llmClient{ modelID: modelID, baseURL: baseURL, numCtx: numCtx, + onStatus: onStatus, httpClient: &http.Client{Timeout: defaultHTTPTimeout}, } } @@ -45,53 +53,49 @@ type ollamaRequest struct { Model string `json:"model"` Messages []Message `json:"messages"` Stream bool `json:"stream"` - Format string `json:"format,omitempty"` Options *ollamaOptions `json:"options,omitempty"` } type ollamaOptions struct { - NumCtx int `json:"num_ctx,omitempty"` + NumCtx int `json:"num_ctx,omitempty"` + Temperature float64 `json:"temperature"` } type ollamaResponse struct { - Message Message `json:"message"` + Message Message `json:"message"` + DoneReason string `json:"done_reason"` } -// prepareOllamaRequest creates and serializes the HTTP request for the Ollama api/chat endpoint. -func (c *llmClient) prepareOllamaRequest(ctx context.Context, systemPrompt string, messages []Message, jsonFormat bool) (*http.Request, error) { - url := strings.TrimSuffix(c.baseURL, "/") + "/api/chat" - +// prepareOllamaRequest serializes the payload for the Ollama api/chat endpoint. +func (c *llmClient) prepareOllamaRequest(systemPrompt, userPrompt string) ([]byte, error) { reqBody := ollamaRequest{ - Model: c.modelID, - Messages: append([]Message{{Role: "system", Content: systemPrompt}}, messages...), - Stream: false, - Options: &ollamaOptions{NumCtx: c.numCtx}, - } - if jsonFormat { - reqBody.Format = "json" + Model: c.modelID, + Messages: []Message{ + {Role: "system", Content: systemPrompt}, + {Role: "user", Content: userPrompt}, + }, + Stream: false, + Options: &ollamaOptions{ + NumCtx: c.numCtx, + Temperature: 0.1, + }, } + return json.Marshal(reqBody) +} - bodyBytes, err := json.Marshal(reqBody) +// CallLLM invokes the LLM via HTTP failing fast without retries. +func (c *llmClient) CallLLM(ctx context.Context, systemPrompt, userPrompt string) (string, error) { + bodyBytes, err := c.prepareOllamaRequest(systemPrompt, userPrompt) if err != nil { - return nil, fmt.Errorf("failed to marshal request: %w", err) + return "", fmt.Errorf("failed to marshal request: %w", err) } + url := strings.TrimSuffix(c.baseURL, "/") + "/api/chat" req, err := http.NewRequestWithContext(ctx, "POST", url, bytes.NewReader(bodyBytes)) - if err != nil { - return nil, err - } - req.Header.Set("Content-Type", "application/json") - - return req, nil -} - -// CallLLM invokes the LLM via HTTP failing fast without retries. -func (c *llmClient) CallLLM(ctx context.Context, systemPrompt string, messages []Message, jsonFormat bool) (string, error) { - - req, err := c.prepareOllamaRequest(ctx, systemPrompt, messages, jsonFormat) if err != nil { return "", err } + req.Header.Set("Content-Type", "application/json") resp, err := c.httpClient.Do(req) if err != nil { @@ -109,6 +113,9 @@ func (c *llmClient) CallLLM(ctx context.Context, systemPrompt string, messages [ if err := json.Unmarshal(respData, &result); err != nil { return "", fmt.Errorf("failed to parse response: %w", err) } + if result.DoneReason == "length" { + c.onStatus("Warning: LLM response was truncated due to context length limit (done_reason: length)") + } return result.Message.Content, nil } diff --git a/internal/engine/constants.go b/internal/engine/constants.go deleted file mode 100644 index 9264f4e..0000000 --- a/internal/engine/constants.go +++ /dev/null @@ -1,17 +0,0 @@ -package engine - -import "time" - -const ( - defaultHTTPTimeout = 10 * time.Minute - maxErrorBodyBytes = 1024 - defaultChunkOverlap = 800 - minNumCtxFloor = 512 - contextWindowAllocRatio = 0.75 - maxCharsMultiplier = 3 - metadataFileName = ".metadata.json" - agentsFileName = "AGENTS.md" - defaultDirPerm = 0755 -) - -type LogEventFunc func(EventType, string) diff --git a/internal/engine/markdown.go b/internal/engine/markdown.go deleted file mode 100644 index 3fbb585..0000000 --- a/internal/engine/markdown.go +++ /dev/null @@ -1,17 +0,0 @@ -package engine - -import ( - "regexp" - "strings" -) - -var markdownFenceRe = regexp.MustCompile("(?s)^\\x60{3,}(?:markdown|json)?\\s*(.*?)\\s*\\x60{3,}$") - -// stripOuterMarkdownFence strips surrounding markdown or json code fences from input strings. -func stripOuterMarkdownFence(content string) string { - trimmed := strings.TrimSpace(content) - if matches := markdownFenceRe.FindStringSubmatch(trimmed); len(matches) > 1 { - return strings.TrimSpace(matches[1]) - } - return trimmed -} diff --git a/internal/engine/orchestrator.go b/internal/engine/orchestrator.go index af0b55e..5115ca7 100644 --- a/internal/engine/orchestrator.go +++ b/internal/engine/orchestrator.go @@ -13,60 +13,128 @@ import ( "github.com/arrase/code-reducer/internal/tools" ) -// EventType defines the type of pipeline events. -type EventType string +const agentsFileName = "AGENTS.md" + +// RunMode represents the operation mode of the documentation pipeline. +type RunMode string const ( - // EventStatus represents progress/status updates. - EventStatus EventType = "status" - // EventError represents execution errors. - EventError EventType = "error" + // ModeInit initializes the project documentation. + ModeInit RunMode = "init" + // ModeUpdate updates the existing project documentation incrementally. + ModeUpdate RunMode = "update" ) -// Event represents a progress or error notification from the pipeline. -type Event struct { - Type EventType - Message string +// StatusFunc is a callback function for reporting pipeline progress and warnings. +type StatusFunc func(string) + +// Run executes the documentation pipeline for the specified mode. +func Run(ctx context.Context, repoRoot string, cfg *config.Config, mode RunMode, onStatus StatusFunc) error { + if onStatus == nil { + onStatus = func(string) {} + } + + if err := tools.EnsureGitignoreHasLockfile(repoRoot); err != nil { + onStatus(fmt.Sprintf("Warning: failed to ensure gitignore has lockfile: %v", err)) + } + + lock, err := security.AcquireLock(repoRoot) + if err != nil { + return fmt.Errorf("failed to acquire repository lock: %w", err) + } + defer lock.Unlock() + + client := newLLMClient(cfg.ModelID, cfg.OllamaBaseURL, cfg.OllamaNumCtx, onStatus) + orch := &orchestrator{client: client} + + switch mode { + case ModeInit: + return orch.RunInit(ctx, repoRoot, cfg, onStatus) + case ModeUpdate: + return orch.RunUpdate(ctx, repoRoot, cfg, onStatus) + default: + return fmt.Errorf("unsupported mode: %s", mode) + } } type orchestrator struct { client llmCaller } -func (o *orchestrator) GenerateStandardDocs(ctx context.Context, repoRoot, docsDir, rootSum string, cfg *config.Config, logEvent LogEventFunc) error { - logEvent(EventStatus, "Step 3: Global Architecture Synthesis...") - archPath := filepath.Join(docsDir, "architecture.md") - archMsg := fmt.Sprintf("Write the global architecture overview (%s/architecture.md) based on the root summary.\n\n%s", docsDir, rootSum) - sysPrompt := cfg.SystemPrompt + "\n" + cfg.ArchitecturePrompt - archDoc, err := o.client.CallLLM(ctx, sysPrompt, []Message{{Role: "user", Content: archMsg}}, false) +func (o *orchestrator) generateDoc(ctx context.Context, repoRoot, relPath, sysPrompt, promptHeader string, contextItems []string, onStatus StatusFunc) error { + redCfg := reductionConfig{ + sysPrompt: sysPrompt, + buildPrompt: func(batch []string) string { + return fmt.Sprintf("%s\n\n%s", promptHeader, strings.Join(batch, "\n\n")) + }, + logMsg: func(batch []string) string { + return fmt.Sprintf("➜ Generating %s (%d items)...", relPath, len(batch)) + }, + errMsg: fmt.Sprintf("failed to generate %s", relPath), + onStatus: onStatus, + } + doc, err := reduceWithLLM(ctx, o.client, contextItems, redCfg) if err != nil { - return fmt.Errorf("failed to generate global architecture: %w", err) + return err } - if err := tools.WriteFileSafely(repoRoot, archPath, []byte(stripOuterMarkdownFence(archDoc))); err != nil { - return fmt.Errorf("failed to write architecture.md: %w", err) + if err := tools.WriteFileSafely(repoRoot, relPath, []byte(doc)); err != nil { + return fmt.Errorf("failed to write %s: %w", relPath, err) } + return nil +} - logEvent(EventStatus, "Step 4: Generating Quickstart...") - qsPath := filepath.Join(docsDir, "quickstart.md") - qsMsg := fmt.Sprintf("Write the %s/quickstart.md page based on this architecture.\n\n%s", docsDir, rootSum) - qsDoc, err := o.client.CallLLM(ctx, sysPrompt, []Message{{Role: "user", Content: qsMsg}}, false) - if err != nil { - return fmt.Errorf("failed to generate quickstart documentation: %w", err) +func buildProjectContextItems(rootSum string, tree *DirNode, cache *MetadataCache) []string { + var allDirs []string + for d := range collectTreeDirs(tree) { + allDirs = append(allDirs, d) } - if err := tools.WriteFileSafely(repoRoot, qsPath, []byte(stripOuterMarkdownFence(qsDoc))); err != nil { - return fmt.Errorf("failed to write quickstart.md: %w", err) + sort.Strings(allDirs) + + items := []string{ + fmt.Sprintf("## Project Directories\n%s", strings.Join(allDirs, ", ")), + fmt.Sprintf("## Root Module Summary\n%s", rootSum), } - return nil + var childNames []string + for name := range tree.Children { + childNames = append(childNames, name) + } + sort.Strings(childNames) + + for _, name := range childNames { + childPath := tree.Children[name].Path + if sum := cache.Modules[childPath]; sum != "" { + items = append(items, fmt.Sprintf("### Top-Level Submodule: %s\n%s", childPath, sum)) + } + } + return items } -func setupPipeline(repoRoot string, cfg *config.Config, logEvent LogEventFunc) (string, []string, *MetadataCache, error) { +func (o *orchestrator) GenerateStandardDocs(ctx context.Context, repoRoot, docsDir, rootSum string, tree *DirNode, cache *MetadataCache, cfg *config.Config, onStatus StatusFunc) error { + contextItems := buildProjectContextItems(rootSum, tree, cache) + + onStatus("Step 3: Global Architecture Synthesis...") + archPath := filepath.Join(docsDir, "architecture.md") + archHeader := fmt.Sprintf("Write the global architecture overview (%s) based on the project context.", archPath) + if err := o.generateDoc(ctx, repoRoot, archPath, cfg.SystemPrompt+"\n"+cfg.ArchitecturePrompt, archHeader, contextItems, onStatus); err != nil { + return err + } + + onStatus("Step 4: Generating Quickstart...") + qsPath := filepath.Join(docsDir, "quickstart.md") + qsHeader := fmt.Sprintf("Write the %s page based on the project context.", qsPath) + return o.generateDoc(ctx, repoRoot, qsPath, cfg.SystemPrompt+"\n"+cfg.QuickstartPrompt, qsHeader, contextItems, onStatus) +} + +func setupPipeline(repoRoot string, cfg *config.Config) (string, []string, *MetadataCache, error) { docsDir := cfg.DocsDir gitignorePatterns, err := tools.LoadGitignore(repoRoot) - if err != nil && !os.IsNotExist(err) { + if err != nil { return "", nil, nil, fmt.Errorf("failed to load .gitignore: %w", err) } - ignores := append(cfg.Ignore, docsDir) + ignores := make([]string, 0, len(cfg.Ignore)+1+len(gitignorePatterns)) + ignores = append(ignores, cfg.Ignore...) + ignores = append(ignores, docsDir) ignores = append(ignores, gitignorePatterns...) cache, err := loadMetadataCache(repoRoot, docsDir) @@ -76,37 +144,39 @@ func setupPipeline(repoRoot string, cfg *config.Config, logEvent LogEventFunc) ( return docsDir, ignores, cache, nil } -func teardownPipeline(repoRoot, docsDir string, cache *MetadataCache, logEvent LogEventFunc, successMsg string) { +func teardownPipeline(repoRoot, docsDir string, cache *MetadataCache, onStatus StatusFunc, successMsg string) { if err := saveMetadataCache(repoRoot, docsDir, cache); err != nil { - logEvent(EventStatus, fmt.Sprintf("Warning: failed to save metadata cache: %v", err)) + onStatus(fmt.Sprintf("Warning: failed to save metadata cache: %v", err)) } - logEvent(EventStatus, successMsg) + onStatus(successMsg) } -func computeFilesHashes(repoRoot string, files []string, logEvent LogEventFunc) map[string]string { - hashes := make(map[string]string) - for _, f := range files { +func hashCodeFiles(repoRoot string, codeFiles []string, onStatus StatusFunc) ([]string, map[string]string) { + var validFiles []string + currentHashes := make(map[string]string, len(codeFiles)) + for _, f := range codeFiles { hash, err := computeSHA256(repoRoot, f) if err != nil { - logEvent(EventStatus, fmt.Sprintf("Warning: failed to compute hash for %s: %v", f, err)) + onStatus(fmt.Sprintf("Warning: failed to compute hash for %s: %v", f, err)) continue } - hashes[f] = hash + validFiles = append(validFiles, f) + currentHashes[f] = hash } - return hashes + return validFiles, currentHashes } -func markAllTreeAffected(tree *DirNode) map[string]bool { - affected := make(map[string]bool) +func collectTreeDirs(tree *DirNode) map[string]bool { + dirs := make(map[string]bool) var walk func(n *DirNode) walk = func(n *DirNode) { - affected[n.Path] = true + dirs[n.Path] = true for _, child := range n.Children { walk(child) } } walk(tree) - return affected + return dirs } func updateAgentGuidelines(repoRoot, docsDir string) error { @@ -146,31 +216,27 @@ This repository contains automatically generated documentation under the %s dire return nil } -func (o *orchestrator) RunInit(ctx context.Context, repoRoot string, cfg *config.Config, onEvent func(Event)) error { - logEvent := makeLogEvent(onEvent) - logEvent(EventStatus, "Starting Map-Reduce pipeline: init") - logEvent(EventStatus, "Step 1: Code Discovery & Building Tree...") +func (o *orchestrator) RunInit(ctx context.Context, repoRoot string, cfg *config.Config, onStatus StatusFunc) error { + onStatus("Starting Map-Reduce pipeline: init") + onStatus("Step 1: Code Discovery & Building Tree...") - docsDir, ignores, cache, err := setupPipeline(repoRoot, cfg, logEvent) + docsDir, ignores, cache, err := setupPipeline(repoRoot, cfg) if err != nil { return err } - cache.StepsHash = computeStepsHash(cfg.ExtractionSteps) + cache.StepsHash = computeStepsHash(cfg) codeFiles, err := tools.DiscoverCodeFiles(repoRoot, ignores) if err != nil { return err } - precalculatedHashes := computeFilesHashes(repoRoot, codeFiles, logEvent) - tree := buildTree(codeFiles) - modulesDir := filepath.Join(repoRoot, docsDir, "modules") - if err := os.MkdirAll(modulesDir, defaultDirPerm); err != nil { - return fmt.Errorf("failed to create modules directory: %w", err) - } + validFiles, precalculatedHashes := hashCodeFiles(repoRoot, codeFiles, onStatus) + tree := buildTree(validFiles) + pruneStaleCache(repoRoot, docsDir, cache, tree, precalculatedHashes) - logEvent(EventStatus, "Step 2: Hierarchical Tree-Merging (Map-Reduce)...") - affectedDirs := markAllTreeAffected(tree) + onStatus("Step 2: Hierarchical Tree-Merging (Map-Reduce)...") + affectedDirs := collectTreeDirs(tree) pState := &pipelineState{ client: o.client, @@ -179,7 +245,7 @@ func (o *orchestrator) RunInit(ctx context.Context, repoRoot string, cfg *config cache: cache, affectedDirs: affectedDirs, precalculatedHashes: precalculatedHashes, - logEvent: logEvent, + onStatus: onStatus, } rootSum, err := synthesizeNode(ctx, pState, tree) @@ -187,68 +253,73 @@ func (o *orchestrator) RunInit(ctx context.Context, repoRoot string, cfg *config return err } - if err := o.GenerateStandardDocs(ctx, repoRoot, docsDir, rootSum, cfg, logEvent); err != nil { + if err := o.GenerateStandardDocs(ctx, repoRoot, docsDir, rootSum, tree, cache, cfg, onStatus); err != nil { return err } - logEvent(EventStatus, fmt.Sprintf("Step 5: Updating %s...", agentsFileName)) + onStatus(fmt.Sprintf("Step 5: Updating %s...", agentsFileName)) if err := updateAgentGuidelines(repoRoot, docsDir); err != nil { return err } - teardownPipeline(repoRoot, docsDir, cache, logEvent, "Pipeline completed successfully!") + teardownPipeline(repoRoot, docsDir, cache, onStatus, "Pipeline completed successfully!") return nil } -func (o *orchestrator) pruneStaleCache(repoRoot string, docsDir string, cache *MetadataCache, tree *DirNode) { - activeDirs := make(map[string]bool) - var collectDirs func(n *DirNode) - collectDirs = func(n *DirNode) { - activeDirs[n.Path] = true - for _, child := range n.Children { - collectDirs(child) +func pruneStaleCache(repoRoot string, docsDir string, cache *MetadataCache, tree *DirNode, currentFiles map[string]string) { + for f := range cache.Files { + if _, exists := currentFiles[f]; !exists { + delete(cache.Files, f) } } - collectDirs(tree) - + activeDirs := collectTreeDirs(tree) for mPath := range cache.Modules { if !activeDirs[mPath] { delete(cache.Modules, mPath) - moduleFile := filepath.Join(docsDir, "modules", toSafeMarkdownFilename(mPath)) - if absModuleFile, err := security.SafeResolve(repoRoot, moduleFile); err == nil { - _ = os.Remove(absModuleFile) - } } } -} -func (o *orchestrator) getAffectedDirs(repoRoot, docsDir string, tree *DirNode, cache *MetadataCache, changes []FileChange) map[string]bool { - affectedDirs := determineAffected(tree, repoRoot, docsDir, cache, changes) - return propagateAffected(tree, affectedDirs) + modulesRoot, err := security.SafeResolve(repoRoot, filepath.Join(docsDir, "modules")) + if err != nil { + return + } + activeDocs := make(map[string]bool, len(activeDirs)) + for mPath := range activeDirs { + activeDocs[filepath.Join(modulesRoot, toSafeMarkdownFilename(mPath))] = true + } + + var subdirs []string + _ = filepath.WalkDir(modulesRoot, func(p string, d os.DirEntry, walkErr error) error { + if walkErr != nil { + return nil + } + if d.IsDir() { + if p != modulesRoot { + subdirs = append(subdirs, p) + } + return nil + } + if !activeDocs[p] { + _ = os.Remove(p) + } + return nil + }) + for i := len(subdirs) - 1; i >= 0; i-- { + _ = os.Remove(subdirs[i]) + } } -func (o *orchestrator) detectFileChanges(repoRoot string, ignores []string, cache *MetadataCache, logEvent LogEventFunc) ([]FileChange, map[string]string, []string, error) { +func detectFileChanges(repoRoot string, ignores []string, cache *MetadataCache, onStatus StatusFunc) ([]FileChange, map[string]string, []string, error) { codeFiles, err := tools.DiscoverCodeFiles(repoRoot, ignores) if err != nil { return nil, nil, nil, err } - var filteredChanges []FileChange - currentFilesMap := make(map[string]string) - var allowedCodeFiles []string - - for _, f := range codeFiles { - hash, err := computeSHA256(repoRoot, f) - if err != nil { - logEvent(EventStatus, fmt.Sprintf("Warning: failed to compute hash for %s: %v", f, err)) - continue - } - currentFilesMap[f] = hash - allowedCodeFiles = append(allowedCodeFiles, f) - } + validFiles, currentHashes := hashCodeFiles(repoRoot, codeFiles, onStatus) - for _, f := range allowedCodeFiles { - currentHash := currentFilesMap[f] + var filteredChanges []FileChange + for _, f := range validFiles { + currentHash := currentHashes[f] cachedEntry, exists := cache.Files[f] if !exists { filteredChanges = append(filteredChanges, FileChange{Path: f, Status: StatusAdded}) @@ -263,60 +334,52 @@ func (o *orchestrator) detectFileChanges(repoRoot string, ignores []string, cach } sort.Strings(cachedFiles) for _, f := range cachedFiles { - if _, exists := currentFilesMap[f]; !exists { + if _, exists := currentHashes[f]; !exists { filteredChanges = append(filteredChanges, FileChange{Path: f, Status: StatusDeleted}) } } - return filteredChanges, currentFilesMap, allowedCodeFiles, nil + return filteredChanges, currentHashes, validFiles, nil } -func (o *orchestrator) RunUpdate(ctx context.Context, repoRoot string, cfg *config.Config, onEvent func(Event)) error { - logEvent := makeLogEvent(onEvent) - logEvent(EventStatus, "Starting Map-Reduce pipeline: update") +func (o *orchestrator) RunUpdate(ctx context.Context, repoRoot string, cfg *config.Config, onStatus StatusFunc) error { + onStatus("Starting Map-Reduce pipeline: update") - docsDir, ignores, cache, err := setupPipeline(repoRoot, cfg, logEvent) + docsDir, ignores, cache, err := setupPipeline(repoRoot, cfg) if err != nil { return err } - currentStepsHash := computeStepsHash(cfg.ExtractionSteps) + currentStepsHash := computeStepsHash(cfg) if cache.StepsHash != currentStepsHash { - logEvent(EventStatus, "Warning: Extraction steps changed. Invalidating metadata cache to force full documentation regeneration.") + onStatus("Warning: Extraction steps changed. Invalidating metadata cache to force full documentation regeneration.") cache.Files = make(map[string]FileCacheEntry) cache.Modules = make(map[string]string) cache.StepsHash = currentStepsHash } - logEvent(EventStatus, "Step 1: Detecting changed files...") + onStatus("Step 1: Detecting changed files...") - filteredChanges, currentFilesMap, allowedCodeFiles, err := o.detectFileChanges(repoRoot, ignores, cache, logEvent) + filteredChanges, currentFilesMap, allowedCodeFiles, err := detectFileChanges(repoRoot, ignores, cache, onStatus) if err != nil { return err } tree := buildTree(allowedCodeFiles) + pruneStaleCache(repoRoot, docsDir, cache, tree, currentFilesMap) - codeFileMap := make(map[string]bool) - for _, f := range allowedCodeFiles { - codeFileMap[f] = true - } - for f := range cache.Files { - if !codeFileMap[f] { - delete(cache.Files, f) - } - } - - o.pruneStaleCache(repoRoot, docsDir, cache, tree) + affectedDirs := propagateAffected(tree, determineAffected(tree, repoRoot, docsDir, cache, filteredChanges)) - affectedDirs := o.getAffectedDirs(repoRoot, docsDir, tree, cache, filteredChanges) + archPath := filepath.Join(docsDir, "architecture.md") + qsPath := filepath.Join(docsDir, "quickstart.md") + missingGlobalDocs := !tools.FileExistsSafely(repoRoot, archPath) || !tools.FileExistsSafely(repoRoot, qsPath) - if len(affectedDirs) == 0 { - logEvent(EventStatus, "No modifications detected. Documentation is up to date.") + if len(affectedDirs) == 0 && !missingGlobalDocs { + onStatus("No modifications detected. Documentation is up to date.") return nil } - logEvent(EventStatus, fmt.Sprintf("Step 2: Hierarchical Tree-Merging (Map-Reduce)... (Affected modules: %d)", len(affectedDirs))) + onStatus(fmt.Sprintf("Step 2: Hierarchical Tree-Merging (Map-Reduce)... (Affected modules: %d)", len(affectedDirs))) pState := &pipelineState{ client: o.client, repoRoot: repoRoot, @@ -324,7 +387,7 @@ func (o *orchestrator) RunUpdate(ctx context.Context, repoRoot string, cfg *conf cache: cache, affectedDirs: affectedDirs, precalculatedHashes: currentFilesMap, - logEvent: logEvent, + onStatus: onStatus, } rootSum, err := synthesizeNode(ctx, pState, tree) @@ -332,29 +395,10 @@ func (o *orchestrator) RunUpdate(ctx context.Context, repoRoot string, cfg *conf return err } - archPath := filepath.Join(docsDir, "architecture.md") - qsPath := filepath.Join(docsDir, "quickstart.md") - archExists := false - qsExists := false - if absArch, err := security.SafeResolve(repoRoot, archPath); err == nil { - if _, err := os.Stat(absArch); err == nil { - archExists = true - } - } - if absQs, err := security.SafeResolve(repoRoot, qsPath); err == nil { - if _, err := os.Stat(absQs); err == nil { - qsExists = true - } - } - - if affectedDirs["."] || !archExists || !qsExists { - if err := o.GenerateStandardDocs(ctx, repoRoot, docsDir, rootSum, cfg, logEvent); err != nil { - return err - } - } else { - logEvent(EventStatus, "Global architecture and quickstart pages are up to date. Skipping regeneration.") + if err := o.GenerateStandardDocs(ctx, repoRoot, docsDir, rootSum, tree, cache, cfg, onStatus); err != nil { + return err } - teardownPipeline(repoRoot, docsDir, cache, logEvent, "Pipeline update completed successfully!") + teardownPipeline(repoRoot, docsDir, cache, onStatus, "Pipeline update completed successfully!") return nil } diff --git a/internal/engine/orchestrator_test.go b/internal/engine/orchestrator_test.go index 1247669..21e1fe8 100644 --- a/internal/engine/orchestrator_test.go +++ b/internal/engine/orchestrator_test.go @@ -2,14 +2,18 @@ package engine import ( "context" + "encoding/json" + "net/http" + "net/http/httptest" "os" "path/filepath" + "strings" "testing" "github.com/arrase/code-reducer/internal/config" ) -func TestMarkAllTreeAffected(t *testing.T) { +func TestCollectTreeDirs(t *testing.T) { tree := &DirNode{ Path: ".", Children: map[string]*DirNode{ @@ -25,21 +29,23 @@ func TestMarkAllTreeAffected(t *testing.T) { }, } - affected := markAllTreeAffected(tree) - if !affected["."] || !affected["cmd"] || !affected["internal"] || !affected["internal/engine"] { - t.Fatalf("expected all nodes to be affected, got %v", affected) + dirs := collectTreeDirs(tree) + if !dirs["."] || !dirs["cmd"] || !dirs["internal"] || !dirs["internal/engine"] { + t.Fatalf("expected all nodes to be collected, got %v", dirs) } } -func TestComputeFilesHashes(t *testing.T) { +func TestHashCodeFiles(t *testing.T) { tmp := t.TempDir() file1 := "file1.txt" if err := os.WriteFile(filepath.Join(tmp, file1), []byte("hello"), 0644); err != nil { t.Fatalf("failed to write test file: %v", err) } - logEvent := func(t EventType, msg string) {} - hashes := computeFilesHashes(tmp, []string{file1, "missing.txt"}, logEvent) + validFiles, hashes := hashCodeFiles(tmp, []string{file1, "missing.txt"}, func(string) {}) + if len(validFiles) != 1 || validFiles[0] != file1 { + t.Fatalf("expected validFiles [%s], got %v", file1, validFiles) + } if hashes[file1] == "" { t.Fatalf("expected hash for file1, got empty") } @@ -70,7 +76,7 @@ func TestUpdateAgentGuidelines(t *testing.T) { } } -func TestOrchestratorRunInit(t *testing.T) { +func TestOrchestratorRunInitAndUpdate(t *testing.T) { repoRoot := t.TempDir() file1 := "main.go" if err := os.WriteFile(filepath.Join(repoRoot, file1), []byte("package main\n\nfunc main() {}\n"), 0644); err != nil { @@ -78,13 +84,156 @@ func TestOrchestratorRunInit(t *testing.T) { } cfg := &config.Config{ - DocsDir: "docs", - ExtractionSteps: config.DefaultExtractionSteps, + DocsDir: "docs", + SystemPrompt: config.DefaultSystemPrompt, + ArchitecturePrompt: config.DefaultArchitecturePrompt, + QuickstartPrompt: config.DefaultQuickstartPrompt, + ExtractionSteps: config.DefaultExtractionSteps, } o := &orchestrator{client: &mockLLMCaller{numCtx: 2048}} - err := o.RunInit(context.Background(), repoRoot, cfg, func(e Event) {}) - if err != nil { + if err := o.RunInit(context.Background(), repoRoot, cfg, func(string) {}); err != nil { t.Fatalf("unexpected error running RunInit: %v", err) } + + if err := o.RunUpdate(context.Background(), repoRoot, cfg, func(string) {}); err != nil { + t.Fatalf("unexpected error running RunUpdate: %v", err) + } + + // Deleting quickstart.md with no code changes should still trigger regeneration on update + qsPath := filepath.Join(repoRoot, "docs", "quickstart.md") + if err := os.Remove(qsPath); err != nil { + t.Fatalf("failed to remove quickstart.md: %v", err) + } + if err := o.RunUpdate(context.Background(), repoRoot, cfg, func(string) {}); err != nil { + t.Fatalf("unexpected error running RunUpdate after deleting quickstart.md: %v", err) + } + if _, err := os.Stat(qsPath); err != nil { + t.Fatalf("expected quickstart.md to be regenerated on RunUpdate, got error: %v", err) + } + + // Orphan module docs and empty directories in docs/modules/ should be cleaned up on update + orphanDir := filepath.Join(repoRoot, "docs", "modules", "removed_pkg") + orphanFile := filepath.Join(orphanDir, "README.md") + if err := os.MkdirAll(orphanDir, 0755); err != nil { + t.Fatalf("failed to create orphan dir: %v", err) + } + if err := os.WriteFile(orphanFile, []byte("stale"), 0644); err != nil { + t.Fatalf("failed to write orphan file: %v", err) + } + if err := o.RunUpdate(context.Background(), repoRoot, cfg, func(string) {}); err != nil { + t.Fatalf("unexpected error running RunUpdate with orphan module: %v", err) + } + if _, err := os.Stat(orphanDir); !os.IsNotExist(err) { + t.Fatalf("expected orphan module directory %s to be removed, got err: %v", orphanDir, err) + } +} + +func TestComputeStepsHash(t *testing.T) { + baseCfg := &config.Config{ + ModelID: "qwen:7b", + SystemPrompt: "sys", + ModuleSynthesisPrompt: "mod", + ArchitecturePrompt: "arch", + QuickstartPrompt: "qs", + FileFactConsolidationPrompt: "fact", + ExtractionSteps: []config.ExtractionStep{{Name: "STEP1", Prompt: "p1"}}, + } + baseHash := computeStepsHash(baseCfg) + + modifiedModel := *baseCfg + modifiedModel.ModelID = "qwen:14b" + if computeStepsHash(&modifiedModel) == baseHash { + t.Errorf("expected hash to change when ModelID changes") + } + + modifiedPrompt := *baseCfg + modifiedPrompt.SystemPrompt = "sys2" + if computeStepsHash(&modifiedPrompt) == baseHash { + t.Errorf("expected hash to change when SystemPrompt changes") + } + + modifiedArch := *baseCfg + modifiedArch.ArchitecturePrompt = "arch2" + if computeStepsHash(&modifiedArch) == baseHash { + t.Errorf("expected hash to change when ArchitecturePrompt changes") + } + + modifiedQS := *baseCfg + modifiedQS.QuickstartPrompt = "qs2" + if computeStepsHash(&modifiedQS) == baseHash { + t.Errorf("expected hash to change when QuickstartPrompt changes") + } + + // Verify field boundary shifts do not collide ("ab"+"c" vs "a"+"bc") + shiftedBoundary := *baseCfg + shiftedBoundary.ExtractionSteps = []config.ExtractionStep{{Name: "STEP", Prompt: "1p1"}} + if computeStepsHash(&shiftedBoundary) == baseHash { + t.Errorf("expected hash to differ when characters shift across step Name/Prompt boundary") + } +} + +func TestLoadMetadataCacheErrorReturnsNil(t *testing.T) { + repoRoot := t.TempDir() + docsDir := "docs" + if err := os.MkdirAll(filepath.Join(repoRoot, docsDir), 0755); err != nil { + t.Fatalf("failed to create docs dir: %v", err) + } + if err := os.WriteFile(filepath.Join(repoRoot, docsDir, metadataFileName), []byte("{invalid-json"), 0644); err != nil { + t.Fatalf("failed to write corrupted metadata: %v", err) + } + + cache, err := loadMetadataCache(repoRoot, docsDir) + if err == nil { + t.Fatalf("expected error on invalid metadata JSON") + } + if cache != nil { + t.Fatalf("expected nil cache on error, got %v", cache) + } +} + +func TestPrepareOllamaRequest(t *testing.T) { + client := newLLMClient("ornith:9b", "http://localhost:11434", 4096, func(string) {}) + payload, err := client.prepareOllamaRequest("sys prompt", "user prompt") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + var req ollamaRequest + if err := json.Unmarshal(payload, &req); err != nil { + t.Fatalf("failed to unmarshal payload: %v", err) + } + if len(req.Messages) != 2 || req.Messages[0].Content != "sys prompt" || req.Messages[1].Content != "user prompt" { + t.Fatalf("unexpected messages: %+v", req.Messages) + } + if req.Options == nil || req.Options.Temperature != 0.1 || req.Options.NumCtx != 4096 { + t.Fatalf("unexpected options: %+v", req.Options) + } +} + +func TestCallLLMDoneReasonLengthWarning(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + resp := ollamaResponse{ + Message: Message{Role: "assistant", Content: "partial output"}, + DoneReason: "length", + } + _ = json.NewEncoder(w).Encode(resp) + })) + defer srv.Close() + + var warnings []string + client := newLLMClient("ornith:9b", srv.URL, 4096, func(msg string) { + warnings = append(warnings, msg) + }) + + out, err := client.CallLLM(context.Background(), "sys", "user") + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if out != "partial output" { + t.Fatalf("expected 'partial output', got %q", out) + } + if len(warnings) != 1 || !strings.Contains(warnings[0], "done_reason: length") { + t.Fatalf("expected truncation warning, got %v", warnings) + } } diff --git a/internal/engine/runner.go b/internal/engine/runner.go deleted file mode 100644 index 10666af..0000000 --- a/internal/engine/runner.go +++ /dev/null @@ -1,68 +0,0 @@ -package engine - -import ( - "context" - "fmt" - - "github.com/arrase/code-reducer/internal/config" - "github.com/arrase/code-reducer/internal/security" - "github.com/arrase/code-reducer/internal/tools" -) - -// Mode represents the operation mode of the documentation pipeline. -type Mode string - -const ( - // ModeInit initializes the project documentation. - ModeInit Mode = "init" - // ModeUpdate updates the existing project documentation incrementally. - ModeUpdate Mode = "update" -) - -// Runner orchestrates the execution of the documentation pipeline. -type Runner struct { - cfg *config.Config -} - -// NewRunner creates a new Runner instance with the given configuration. -func NewRunner(cfg *config.Config) *Runner { - return &Runner{ - cfg: cfg, - } -} - -// Run executes the documentation pipeline for the specified mode. -func (r *Runner) Run(ctx context.Context, repoRoot string, mode Mode, onEvent func(Event)) error { - logEvent := makeLogEvent(onEvent) - - // 1. Ensure lockfile is in gitignore - if err := tools.EnsureGitignoreHasLockfile(repoRoot); err != nil { - logEvent(EventStatus, fmt.Sprintf("Warning: failed to ensure gitignore has lockfile: %v", err)) - } - - // 2. Acquire repository lock - lock, err := security.AcquireLock(repoRoot) - if err != nil { - return fmt.Errorf("failed to acquire repository lock: %w", err) - } - defer lock.Unlock() - - // 3. Instantiate LLM Client & Orchestrator - client := newLLMClient(r.cfg.ModelID, r.cfg.OllamaBaseURL, r.cfg.OllamaNumCtx) - orch := &orchestrator{client: client} - - // 4. Run the documentation pipeline - if mode == ModeInit { - if err := orch.RunInit(ctx, repoRoot, r.cfg, onEvent); err != nil { - return err - } - } else if mode == ModeUpdate { - if err := orch.RunUpdate(ctx, repoRoot, r.cfg, onEvent); err != nil { - return err - } - } else { - return fmt.Errorf("unsupported mode: %s", mode) - } - - return nil -} diff --git a/internal/engine/synthesize.go b/internal/engine/synthesize.go index 26cc713..d165201 100644 --- a/internal/engine/synthesize.go +++ b/internal/engine/synthesize.go @@ -2,17 +2,25 @@ package engine import ( "context" - "crypto/sha256" - "encoding/hex" "fmt" + "path" "path/filepath" "sort" "strings" + "unicode/utf8" "github.com/arrase/code-reducer/internal/config" "github.com/arrase/code-reducer/internal/tools" ) +// toSafeMarkdownFilename converts a module path to a safe filename for markdown docs. +func toSafeMarkdownFilename(modulePath string) string { + if modulePath == "." || modulePath == "" { + return "README.md" + } + return filepath.Join(modulePath, "README.md") +} + type pipelineState struct { client llmCaller repoRoot string @@ -20,35 +28,43 @@ type pipelineState struct { cache *MetadataCache affectedDirs map[string]bool precalculatedHashes map[string]string - logEvent LogEventFunc + onStatus StatusFunc +} + +func buildExtractionPrompt(filePath, siblingList, chunk string, step config.ExtractionStep) string { + return fmt.Sprintf("Target File: %s\nDirectory Files: %s\n---\n%s\n---\nTask (%s): %s\nReminder: Be concise, extract only verifiable facts from the code above, do not copy code blocks, and output 'None' if not applicable.", + filePath, siblingList, chunk, step.Name, step.Prompt) } -func extractFileFacts(ctx context.Context, p *pipelineState, f string, nodePath string, fileLimit int) (string, error) { - fileHash, hashOk := p.precalculatedHashes[f] - cachedEntry, cacheExists := p.cache.Files[f] +func maxExtractionOverhead(sysPrompt, filePath, siblingList string, steps []config.ExtractionStep) string { + maxWrapper := "" + maxRunes := 0 + for _, step := range steps { + wrapper := buildExtractionPrompt(filePath, siblingList, "", step) + if runes := utf8.RuneCountInString(wrapper); runes > maxRunes { + maxRunes = runes + maxWrapper = wrapper + } + } + return sysPrompt + maxWrapper +} - // 1. Fast path: cache hit with precalculated hash - if hashOk && cacheExists && cachedEntry.SHA256 == fileHash { +func extractFileFacts(ctx context.Context, p *pipelineState, f string, siblingFiles []string) (string, error) { + fileHash := p.precalculatedHashes[f] + if cachedEntry, cacheExists := p.cache.Files[f]; cacheExists && cachedEntry.SHA256 == fileHash { return cachedEntry.Facts, nil } - // 2. Read file content once contentBytes, err := tools.ReadFileSafely(p.repoRoot, f) if err != nil { - p.logEvent(EventStatus, fmt.Sprintf("Warning: failed to read file %s: %v", f, err)) + p.onStatus(fmt.Sprintf("Warning: failed to read file %s: %v", f, err)) return "", nil } - // 3. Compute hash if not precalculated and check cache again - if !hashOk { - hashSum := sha256.Sum256(contentBytes) - fileHash = hex.EncodeToString(hashSum[:]) - if cacheExists && cachedEntry.SHA256 == fileHash { - return cachedEntry.Facts, nil - } - } + siblingList := strings.Join(siblingFiles, ", ") + overhead := maxExtractionOverhead(p.cfg.SystemPrompt, f, siblingList, p.cfg.ExtractionSteps) + fileLimit := calculateFileLimit(p.client.NumCtx(), overhead) - // 4. Extract facts contentStr := string(contentBytes) overlap := defaultChunkOverlap if overlap > fileLimit/4 { @@ -59,35 +75,36 @@ func extractFileFacts(ctx context.Context, p *pipelineState, f string, nodePath return "", fmt.Errorf("failed to chunk file %s: %w", f, err) } - var factsBuilder strings.Builder - for i, step := range p.cfg.ExtractionSteps { - var stepFacts []string - for chunkIdx, chunk := range chunks { - chunkMsg := "" - if len(chunks) > 1 { - chunkMsg = fmt.Sprintf(" (Chunk %d of %d)", chunkIdx+1, len(chunks)) - } - p.logEvent(EventStatus, fmt.Sprintf("➜ Extracting file (Step %d/%d - %s)%s: %s", i+1, len(p.cfg.ExtractionSteps), step.Name, chunkMsg, f)) + stepFacts := make([][]string, len(p.cfg.ExtractionSteps)) + for chunkIdx, chunk := range chunks { + chunkMsg := "" + if len(chunks) > 1 { + chunkMsg = fmt.Sprintf(" (Chunk %d of %d)", chunkIdx+1, len(chunks)) + } + for i, step := range p.cfg.ExtractionSteps { + p.onStatus(fmt.Sprintf("➜ Extracting file (Step %d/%d - %s)%s: %s", i+1, len(p.cfg.ExtractionSteps), step.Name, chunkMsg, f)) - systemPrompt := p.cfg.SystemPrompt + "\n" + step.Prompt - userContent := fmt.Sprintf("File: %s%s inside Module: %s\n```\n%s\n```", filepath.Base(f), chunkMsg, nodePath, chunk) - res, err := p.client.CallLLM(ctx, systemPrompt, []Message{{Role: "user", Content: userContent}}, false) + userPrompt := buildExtractionPrompt(f, siblingList, chunk, step) + res, err := p.client.CallLLM(ctx, p.cfg.SystemPrompt, userPrompt) if err != nil { return "", fmt.Errorf("LLM error extracting %s for %s: %w", step.Name, f, err) } - stepFacts = append(stepFacts, stripOuterMarkdownFence(res)) + stepFacts[i] = append(stepFacts[i], stripOuterMarkdownFence(res)) } + } - consolidatedFact, err := reduceFileFacts(ctx, p.client, f, step.Name, stepFacts, p.cfg, p.logEvent) + var factsBuilder strings.Builder + for i, step := range p.cfg.ExtractionSteps { + consolidatedFact, err := reduceFileFacts(ctx, p.client, f, step.Name, stepFacts[i], p.cfg, p.onStatus) if err != nil { return "", err } - - factsBuilder.WriteString(fmt.Sprintf("#### [%s]\n%s\n\n", step.Name, consolidatedFact)) + if consolidatedFact != "" { + factsBuilder.WriteString(fmt.Sprintf("#### [%s]\n%s\n\n", step.Name, consolidatedFact)) + } } facts := strings.TrimSpace(factsBuilder.String()) - // 5. Update cache p.cache.Files[f] = FileCacheEntry{ SHA256: fileHash, Facts: facts, @@ -96,57 +113,43 @@ func extractFileFacts(ctx context.Context, p *pipelineState, f string, nodePath return facts, nil } -func calculateFileLimit(numCtx int) int { - if numCtx < minNumCtxFloor { - numCtx = minNumCtxFloor - } - return int(float64(numCtx*4) * contextWindowAllocRatio) -} - -func synthesizeChildren(ctx context.Context, p *pipelineState, node *DirNode) (map[string]string, []string, error) { +func synthesizeChildren(ctx context.Context, p *pipelineState, node *DirNode) ([]string, error) { var childNames []string for name := range node.Children { childNames = append(childNames, name) } sort.Strings(childNames) - childSummaries := make(map[string]string) + var subsystems []string for _, name := range childNames { sum, err := synthesizeNode(ctx, p, node.Children[name]) if err != nil { - return nil, nil, err + return nil, err } if sum != "" { - childSummaries[name] = sum + subsystems = append(subsystems, fmt.Sprintf("### Subsystem: %s\n%s", name, sum)) } } - return childSummaries, childNames, nil + return subsystems, nil } -func collectComponents(ctx context.Context, p *pipelineState, node *DirNode, childSummaries map[string]string, childNames []string) ([]string, error) { - fileLimit := calculateFileLimit(p.client.NumCtx()) - +func collectComponents(ctx context.Context, p *pipelineState, node *DirNode, subsystems []string) ([]string, error) { var components []string for _, f := range node.Files { if err := ctx.Err(); err != nil { return nil, err } - facts, err := extractFileFacts(ctx, p, f, node.Path, fileLimit) + facts, err := extractFileFacts(ctx, p, f, node.Files) if err != nil { return nil, err } if facts != "" { - components = append(components, fmt.Sprintf("### File: %s\n%s", filepath.Base(f), facts)) + components = append(components, fmt.Sprintf("### File: %s\n%s", path.Base(f), facts)) } } - for _, childName := range childNames { - if sum := childSummaries[childName]; sum != "" { - components = append(components, fmt.Sprintf("### Subsystem: %s\n%s", childName, sum)) - } - } - return components, nil + return append(components, subsystems...), nil } func synthesizeNode(ctx context.Context, p *pipelineState, node *DirNode) (string, error) { @@ -156,27 +159,29 @@ func synthesizeNode(ctx context.Context, p *pipelineState, node *DirNode) (strin // If this node (and all descendants) is NOT affected, reuse cached summary! if !p.affectedDirs[node.Path] && p.cache.Modules[node.Path] != "" { - p.logEvent(EventStatus, fmt.Sprintf("➜ Reusing cached summary for directory: %s", node.Path)) + p.onStatus(fmt.Sprintf("➜ Reusing cached summary for directory: %s", node.Path)) return p.cache.Modules[node.Path], nil } - childSummaries, childNames, err := synthesizeChildren(ctx, p, node) + subsystems, err := synthesizeChildren(ctx, p, node) if err != nil { return "", err } - components, err := collectComponents(ctx, p, node, childSummaries, childNames) + components, err := collectComponents(ctx, p, node, subsystems) if err != nil { return "", err } + modulePath := filepath.Join(p.cfg.DocsDir, "modules", toSafeMarkdownFilename(node.Path)) if len(components) == 0 { - p.cache.Modules[node.Path] = "" + delete(p.cache.Modules, node.Path) + _ = tools.RemoveFileSafely(p.repoRoot, modulePath) return "", nil } - p.logEvent(EventStatus, fmt.Sprintf("➜ Synthesizing directory: %s (%d total components)", node.Path, len(components))) - finalSum, err := reduceInChunks(ctx, p.client, node.Path, components, p.cfg, p.logEvent) + p.onStatus(fmt.Sprintf("➜ Synthesizing directory: %s (%d total components)", node.Path, len(components))) + finalSum, err := reduceInChunks(ctx, p.client, node.Path, components, p.cfg, p.onStatus) if err != nil { return "", err } @@ -184,10 +189,13 @@ func synthesizeNode(ctx context.Context, p *pipelineState, node *DirNode) (strin // Update module cache p.cache.Modules[node.Path] = finalSum - modulePath := filepath.Join(p.cfg.DocsDir, "modules", toSafeMarkdownFilename(node.Path)) if err := tools.WriteFileSafely(p.repoRoot, modulePath, []byte(finalSum)); err != nil { return "", fmt.Errorf("failed to write module documentation for %s: %w", node.Path, err) } + if err := saveMetadataCache(p.repoRoot, p.cfg.DocsDir, p.cache); err != nil { + return "", err + } + return finalSum, nil } diff --git a/internal/engine/synthesize_test.go b/internal/engine/synthesize_test.go index b23b7cf..7290f77 100644 --- a/internal/engine/synthesize_test.go +++ b/internal/engine/synthesize_test.go @@ -4,16 +4,19 @@ import ( "context" "os" "path/filepath" + "strings" "testing" "github.com/arrase/code-reducer/internal/config" ) type mockLLMCaller struct { - numCtx int + numCtx int + userPrompts []string } -func (m *mockLLMCaller) CallLLM(ctx context.Context, systemPrompt string, messages []Message, jsonFormat bool) (string, error) { +func (m *mockLLMCaller) CallLLM(ctx context.Context, systemPrompt, userPrompt string) (string, error) { + m.userPrompts = append(m.userPrompts, userPrompt) return "Mocked LLM summary response", nil } @@ -55,7 +58,7 @@ func TestSynthesizeNode(t *testing.T) { cache: cache, affectedDirs: affectedDirs, precalculatedHashes: hashes, - logEvent: func(t EventType, msg string) {}, + onStatus: func(string) {}, } // 1. Empty node @@ -94,4 +97,120 @@ func TestSynthesizeNode(t *testing.T) { if res == "" { t.Fatalf("expected non-empty synthesis result") } + if !IsInitialized(repoRoot, docsDir) { + t.Fatalf("expected metadata cache to be saved incrementally") + } +} + +func TestExtractFileFactsMultiChunkKVCacheOrder(t *testing.T) { + repoRoot := t.TempDir() + sampleFile := "large.go" + content := strings.Repeat("func partA() {}\n", 60) + strings.Repeat("func partB() {}\n", 60) + if err := os.WriteFile(filepath.Join(repoRoot, sampleFile), []byte(content), 0644); err != nil { + t.Fatalf("failed to write sample file: %v", err) + } + + mock := &mockLLMCaller{numCtx: 500} + cfg := &config.Config{ + SystemPrompt: "sys", + ExtractionSteps: config.DefaultExtractionSteps, + } + p := &pipelineState{ + client: mock, + repoRoot: repoRoot, + cfg: cfg, + cache: newEmptyCache(), + precalculatedHashes: map[string]string{sampleFile: "hash"}, + onStatus: func(string) {}, + } + + _, err := extractFileFacts(context.Background(), p, sampleFile, []string{sampleFile}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + numSteps := len(cfg.ExtractionSteps) + if len(mock.userPrompts) < numSteps*2 { + t.Fatalf("expected at least %d prompts for multi-chunk extraction, got %d", numSteps*2, len(mock.userPrompts)) + } + + // Verify all 4 extraction steps for chunk 0 run consecutively with the exact same code prefix + prefixEnd := strings.Index(mock.userPrompts[0], "\n---\nTask (") + if prefixEnd == -1 { + t.Fatalf("unexpected prompt format: %s", mock.userPrompts[0]) + } + chunk0Prefix := mock.userPrompts[0][:prefixEnd] + for i := 1; i < numSteps; i++ { + if !strings.HasPrefix(mock.userPrompts[i], chunk0Prefix) { + t.Errorf("step %d did not share chunk 0 prefix for Ollama KV cache reuse", i) + } + } +} + +type scriptedLLMCaller struct { + numCtx int + responses []string + calls int +} + +func (s *scriptedLLMCaller) CallLLM(_ context.Context, _, _ string) (string, error) { + idx := s.calls + s.calls++ + if idx < len(s.responses) { + return s.responses[idx], nil + } + return "None", nil +} + +func (s *scriptedLLMCaller) NumCtx() int { + return s.numCtx +} + +func TestExtractFileFactsNoneShortCircuit(t *testing.T) { + repoRoot := t.TempDir() + sampleFile := "stateless.go" + content := strings.Repeat("func helper() {}\n", 60) + strings.Repeat("func helper2() {}\n", 60) + if err := os.WriteFile(filepath.Join(repoRoot, sampleFile), []byte(content), 0644); err != nil { + t.Fatalf("failed to write sample file: %v", err) + } + + cfg := &config.Config{ + SystemPrompt: "sys", + ExtractionSteps: []config.ExtractionStep{ + {Name: "API_SIGNATURES", Prompt: "extract api"}, + {Name: "STATE_AND_CONCURRENCY", Prompt: "extract state"}, + }, + } + + // 2 chunks x 2 steps = 4 extraction calls: + // Chunk 0: API_SIGNATURES returns "- func Helper()", STATE_AND_CONCURRENCY returns "None." + // Chunk 1: API_SIGNATURES returns "- None", STATE_AND_CONCURRENCY returns "None" + // Neither step should trigger a consolidation LLM call (total calls must be 4, not 5 or 6). + caller := &scriptedLLMCaller{ + numCtx: 500, + responses: []string{"- func Helper()", "None.", "- None", "None"}, + } + + p := &pipelineState{ + client: caller, + repoRoot: repoRoot, + cfg: cfg, + cache: newEmptyCache(), + precalculatedHashes: map[string]string{sampleFile: "hash"}, + onStatus: func(string) {}, + } + + facts, err := extractFileFacts(context.Background(), p, sampleFile, []string{sampleFile}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if caller.calls != 4 { + t.Fatalf("expected exactly 4 extraction LLM calls (0 consolidation calls), got %d", caller.calls) + } + if strings.Contains(facts, "STATE_AND_CONCURRENCY") || strings.Contains(facts, "None") { + t.Errorf("expected None step to be omitted from facts, got:\n%s", facts) + } + if !strings.Contains(facts, "#### [API_SIGNATURES]\n- func Helper()") { + t.Errorf("expected API_SIGNATURES fact to be preserved, got:\n%s", facts) + } } diff --git a/internal/engine/tree.go b/internal/engine/tree.go index dae9fb5..3ff800e 100644 --- a/internal/engine/tree.go +++ b/internal/engine/tree.go @@ -1,12 +1,11 @@ package engine import ( - "os" "path" "path/filepath" "strings" - "github.com/arrase/code-reducer/internal/security" + "github.com/arrase/code-reducer/internal/tools" ) type ChangeStatus string @@ -24,13 +23,10 @@ type FileChange struct { func propagateAffected(node *DirNode, affectedDirs map[string]bool) map[string]bool { newAffected := make(map[string]bool) - for k, v := range affectedDirs { - newAffected[k] = v - } var propagate func(*DirNode) bool propagate = func(n *DirNode) bool { - isAffected := newAffected[n.Path] + isAffected := affectedDirs[n.Path] for _, child := range n.Children { if propagate(child) { isAffected = true @@ -57,14 +53,7 @@ func isNodeAffected(n *DirNode, changedFiles map[string]bool, repoRoot, docsDir } modulePath := filepath.Join(docsDir, "modules", toSafeMarkdownFilename(n.Path)) - absModulePath, err := security.SafeResolve(repoRoot, modulePath) - if err == nil { - if _, err := os.Stat(absModulePath); os.IsNotExist(err) { - return true - } - } - - return false + return !tools.FileExistsSafely(repoRoot, modulePath) } func determineAffected(node *DirNode, repoRoot, docsDir string, cache *MetadataCache, filteredChanges []FileChange) map[string]bool { @@ -73,7 +62,12 @@ func determineAffected(node *DirNode, repoRoot, docsDir string, cache *MetadataC for _, c := range filteredChanges { changedFiles[c.Path] = true if c.Status == StatusDeleted { - affectedDirs[path.Dir(c.Path)] = true + for d := path.Dir(c.Path); ; d = path.Dir(d) { + affectedDirs[d] = true + if d == "." { + break + } + } } } diff --git a/internal/engine/tree_test.go b/internal/engine/tree_test.go index 1b58cc6..10616da 100644 --- a/internal/engine/tree_test.go +++ b/internal/engine/tree_test.go @@ -108,4 +108,16 @@ func TestDetermineAndPropagateAffected(t *testing.T) { if propagated["cmd"] { t.Errorf("expected cmd not to be affected after propagation") } + + // Deleted file in a removed subdirectory should propagate to existing ancestors in tree + deletedSubdirChanges := []FileChange{ + {Path: "internal/removed/old.go", Status: StatusDeleted}, + } + affectedDeleted := propagateAffected(tree, determineAffected(tree, tempDir, "docs", cache, deletedSubdirChanges)) + if !affectedDeleted["internal"] || !affectedDeleted["."] { + t.Errorf("expected internal and root to be affected when a child subdirectory is deleted, got %v", affectedDeleted) + } + if affectedDeleted["internal/removed"] { + t.Errorf("did not expect non-existent directory internal/removed in propagated map") + } } diff --git a/internal/engine/utils.go b/internal/engine/utils.go deleted file mode 100644 index cb3a93c..0000000 --- a/internal/engine/utils.go +++ /dev/null @@ -1,21 +0,0 @@ -package engine - -import ( - "path/filepath" -) - -// toSafeMarkdownFilename converts a module path to a safe filename for markdown docs. -func toSafeMarkdownFilename(modulePath string) string { - if modulePath == "." || modulePath == "" { - return "README.md" - } - return filepath.Join(modulePath, "README.md") -} - -func makeLogEvent(onEvent func(Event)) LogEventFunc { - return func(t EventType, m string) { - if onEvent != nil { - onEvent(Event{Type: t, Message: m}) - } - } -} diff --git a/internal/security/errors.go b/internal/security/errors.go deleted file mode 100644 index 290e748..0000000 --- a/internal/security/errors.go +++ /dev/null @@ -1,10 +0,0 @@ -package security - -import "errors" - -var ( - // ErrPathTraversal is returned when a path resolves outside the repository root. - ErrPathTraversal = errors.New("security violation: path traversal detected") - // ErrLockHeld is returned when another code-reducer process holds the file lock. - ErrLockHeld = errors.New("lock is already held by another process") -) diff --git a/internal/security/security.go b/internal/security/security.go index a381cbf..595bf27 100644 --- a/internal/security/security.go +++ b/internal/security/security.go @@ -1,6 +1,7 @@ package security import ( + "errors" "fmt" "os" "path/filepath" @@ -13,6 +14,13 @@ const ( defaultFilePerm = 0644 ) +var ( + // ErrPathTraversal is returned when a path resolves outside the repository root. + ErrPathTraversal = errors.New("security violation: path traversal detected") + // ErrLockHeld is returned when another code-reducer process holds the file lock. + ErrLockHeld = errors.New("lock is already held by another process") +) + // SafeResolve cleans the input path and ensures it lies strictly inside the repository. // It resolves symlinks on the existing ancestor parts to prevent path traversal via symlinks. func SafeResolve(repoRoot, inputPath string) (string, error) { @@ -42,7 +50,7 @@ func SafeResolve(repoRoot, inputPath string) (string, error) { if parent == current { break } - nonExistentSuffix = append([]string{filepath.Base(current)}, nonExistentSuffix...) + nonExistentSuffix = append(nonExistentSuffix, filepath.Base(current)) current = parent } @@ -53,8 +61,10 @@ func SafeResolve(repoRoot, inputPath string) (string, error) { } // Rebuild the resolved full path - parts := append([]string{resolvedAncestor}, nonExistentSuffix...) - resolvedPath := filepath.Join(parts...) + resolvedPath := resolvedAncestor + for i := len(nonExistentSuffix) - 1; i >= 0; i-- { + resolvedPath = filepath.Join(resolvedPath, nonExistentSuffix[i]) + } // Verify that resolvedPath is inside resolvedRoot rel, err := filepath.Rel(resolvedRoot, resolvedPath) @@ -82,8 +92,7 @@ func (l *SimpleLock) Unlock() error { } l.closed = true - var err error - err = l.file.Close() + err := l.file.Close() if removeErr := os.Remove(l.lockPath); removeErr != nil && !os.IsNotExist(removeErr) { if err == nil { err = removeErr @@ -109,16 +118,9 @@ func AcquireLock(repoRoot string) (*SimpleLock, error) { } if _, err := f.Write([]byte(fmt.Sprintf("%d\n", os.Getpid()))); err != nil { - errClose := f.Close() - errRemove := os.Remove(lockPath) - var errMsg string - if errClose != nil { - errMsg += fmt.Sprintf(", close failed: %v", errClose) - } - if errRemove != nil { - errMsg += fmt.Sprintf(", remove failed: %v", errRemove) - } - return nil, fmt.Errorf("failed to write pid to lockfile%s: %w", errMsg, err) + _ = f.Close() + _ = os.Remove(lockPath) + return nil, fmt.Errorf("failed to write pid to lockfile: %w", err) } return &SimpleLock{lockPath: lockPath, file: f}, nil diff --git a/internal/tools/file_tools.go b/internal/tools/file_tools.go index 5ba5e0a..58056f9 100644 --- a/internal/tools/file_tools.go +++ b/internal/tools/file_tools.go @@ -1,7 +1,7 @@ package tools import ( - "bufio" + "bytes" "fmt" "os" "path/filepath" @@ -16,19 +16,80 @@ const ( defaultFilePerm = 0644 ) -// ReadFileSafely resolves the virtual path inside the repository and reads the file content. -func ReadFileSafely(repoRoot, virtualPath string) ([]byte, error) { +var ( + skippedFileNames = map[string]bool{ + "go.sum": true, + "package-lock.json": true, + "pnpm-lock.yaml": true, + "mkdocs.yml": true, + "mkdocs.yaml": true, + "sonar-project.properties": true, + "LICENSE": true, + "Makefile": true, + "Dockerfile": true, + } + + skippedExtensions = map[string]bool{ + ".lock": true, + ".md": true, + ".txt": true, + ".svg": true, + ".png": true, + ".jpg": true, + ".jpeg": true, + ".gif": true, + ".ico": true, + ".pdf": true, + ".bin": true, + ".exe": true, + ".so": true, + ".dylib": true, + ".dll": true, + ".wasm": true, + ".pyc": true, + } + + skippedTestSuffixes = []string{ + "_test.go", + ".test.js", + ".test.ts", + ".test.tsx", + ".spec.js", + ".spec.ts", + ".spec.tsx", + "_test.py", + } +) + +// FileExistsSafely resolves the virtual path inside the repository and checks if the file exists. +func FileExistsSafely(repoRoot, virtualPath string) bool { safePath, err := security.SafeResolve(repoRoot, virtualPath) if err != nil { - return nil, err + return false } + _, err = os.Stat(safePath) + return err == nil +} - content, err := os.ReadFile(safePath) +// RemoveFileSafely resolves the virtual path inside the repository and removes the file, ignoring if it does not exist. +func RemoveFileSafely(repoRoot, virtualPath string) error { + safePath, err := security.SafeResolve(repoRoot, virtualPath) if err != nil { - return nil, fmt.Errorf("failed to read file content: %w", err) + return err + } + if err := os.Remove(safePath); err != nil && !os.IsNotExist(err) { + return err } + return nil +} - return content, nil +// ReadFileSafely resolves the virtual path inside the repository and reads the file content. +func ReadFileSafely(repoRoot, virtualPath string) ([]byte, error) { + safePath, err := security.SafeResolve(repoRoot, virtualPath) + if err != nil { + return nil, err + } + return os.ReadFile(safePath) } // WriteFileAtomic writes data to a file atomically by first writing to a temporary file @@ -115,26 +176,23 @@ func EnsureGitignoreHasLockfile(repoRoot string) error { // LoadGitignore reads the .gitignore file from repoRoot and returns its active ignore patterns. func LoadGitignore(repoRoot string) ([]string, error) { - gitignorePath := filepath.Join(repoRoot, ".gitignore") - file, err := os.Open(gitignorePath) + data, err := ReadFileSafely(repoRoot, ".gitignore") if err != nil { if os.IsNotExist(err) { - return nil, nil // No .gitignore file is not an error + return nil, nil } return nil, err } - defer file.Close() var patterns []string - scanner := bufio.NewScanner(file) - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) + for _, rawLine := range strings.Split(string(data), "\n") { + line := strings.TrimSpace(rawLine) if line == "" || strings.HasPrefix(line, "#") { continue } patterns = append(patterns, line) } - return patterns, scanner.Err() + return patterns, nil } // ShouldIgnoreFile checks if a file (specified by relative path) is ignored. @@ -142,7 +200,7 @@ func ShouldIgnoreFile(relPath string, gitIgnore *ignore.GitIgnore) bool { slashRelPath := filepath.ToSlash(relPath) // 1. Check user-defined ignores (config + gitignore) - if gitIgnore != nil && gitIgnore.MatchesPath(slashRelPath) { + if gitIgnore.MatchesPath(slashRelPath) { return true } @@ -157,6 +215,33 @@ func ShouldIgnoreFile(relPath string, gitIgnore *ignore.GitIgnore) bool { return false } +func shouldSkipCodeFile(name string) bool { + if skippedFileNames[name] || skippedExtensions[strings.ToLower(filepath.Ext(name))] { + return true + } + if strings.HasPrefix(name, "test_") && strings.HasSuffix(name, ".py") { + return true + } + for _, suffix := range skippedTestSuffixes { + if strings.HasSuffix(name, suffix) { + return true + } + } + return false +} + +func isBinaryFile(path string) bool { + f, err := os.Open(path) + if err != nil { + return false + } + defer f.Close() + + var buf [512]byte + n, _ := f.Read(buf[:]) + return bytes.IndexByte(buf[:n], 0) != -1 +} + // DiscoverCodeFiles recursively walks the codebase to find high-signal source files. // It ignores build, dependency, and output files, as well as any paths in the custom ignores list. func DiscoverCodeFiles(repoRoot string, ignores []string) ([]string, error) { @@ -180,14 +265,13 @@ func DiscoverCodeFiles(repoRoot string, ignores []string) ([]string, error) { slashRel := filepath.ToSlash(rel) if d.IsDir() { - name := d.Name() - if strings.HasPrefix(name, ".") || strings.HasSuffix(name, ".egg-info") || (gitIgnore != nil && gitIgnore.MatchesPath(slashRel)) { + if ShouldIgnoreFile(slashRel, gitIgnore) { return filepath.SkipDir } return nil } - if ShouldIgnoreFile(slashRel, gitIgnore) { + if !d.Type().IsRegular() || ShouldIgnoreFile(slashRel, gitIgnore) || shouldSkipCodeFile(d.Name()) || isBinaryFile(path) { return nil } diff --git a/internal/tools/file_tools_test.go b/internal/tools/file_tools_test.go index eee782c..9a4f8ba 100644 --- a/internal/tools/file_tools_test.go +++ b/internal/tools/file_tools_test.go @@ -27,6 +27,44 @@ func TestWriteFileSafely(t *testing.T) { } } +func TestFileExistsAndRemoveSafely(t *testing.T) { + repoRoot := t.TempDir() + + if tools.FileExistsSafely(repoRoot, "src/app.go") { + t.Fatal("Expected FileExistsSafely to be false for missing file") + } + + if err := tools.WriteFileSafely(repoRoot, "src/app.go", []byte("package src")); err != nil { + t.Fatalf("WriteFileSafely() failed: %v", err) + } + + if !tools.FileExistsSafely(repoRoot, "src/app.go") { + t.Fatal("Expected FileExistsSafely to be true for existing file") + } + + if tools.FileExistsSafely(repoRoot, "../outside.go") { + t.Fatal("Expected FileExistsSafely to be false on path traversal") + } + + if err := tools.RemoveFileSafely(repoRoot, "src/app.go"); err != nil { + t.Fatalf("RemoveFileSafely() failed: %v", err) + } + + if tools.FileExistsSafely(repoRoot, "src/app.go") { + t.Fatal("Expected FileExistsSafely to be false after removal") + } + + // Removing again should be a no-op + if err := tools.RemoveFileSafely(repoRoot, "src/app.go"); err != nil { + t.Fatalf("RemoveFileSafely() on non-existent file failed: %v", err) + } + + // Path traversal should return an error + if err := tools.RemoveFileSafely(repoRoot, "../outside.go"); err == nil { + t.Fatal("Expected RemoveFileSafely to error on path traversal") + } +} + func TestLoadGitignore(t *testing.T) { repoRoot := t.TempDir() @@ -83,7 +121,6 @@ func TestShouldIgnoreFile(t *testing.T) { func TestDiscoverCodeFiles(t *testing.T) { repoRoot := t.TempDir() - // Create a structure dirs := []string{ "src", "src/.hidden", @@ -95,19 +132,39 @@ func TestDiscoverCodeFiles(t *testing.T) { } } - files := []string{ + textFiles := []string{ "src/main.go", "src/utils.go", - "src/.hidden/secret.txt", - "build/output.bin", + "src/main_test.go", + "src/app.test.js", + "src/app.spec.ts", + "src/test_runner.py", + "src/runner_test.py", + "src/.hidden/secret.go", + "build/gen.go", "test.log", - } - for _, f := range files { - if err := os.WriteFile(filepath.Join(repoRoot, f), []byte("test"), 0644); err != nil { + "go.sum", + "package-lock.json", + "Makefile", + "Dockerfile", + "README.md", + "notes.txt", + "logo.svg", + } + for _, f := range textFiles { + if err := os.WriteFile(filepath.Join(repoRoot, f), []byte("package main"), 0644); err != nil { t.Fatal(err) } } + // Binary file with a null byte inside first 512 bytes (but valid code extension) + if err := os.WriteFile(filepath.Join(repoRoot, "src/compiled_blob.dat"), []byte{'a', 'b', 0, 'c'}, 0644); err != nil { + t.Fatal(err) + } + + // Symlink to a code file should be skipped + _ = os.Symlink(filepath.Join(repoRoot, "src/main.go"), filepath.Join(repoRoot, "src/symlink.go")) + ignores := []string{"*.log", "build/"} discovered, err := tools.DiscoverCodeFiles(repoRoot, ignores) if err != nil { @@ -119,8 +176,6 @@ func TestDiscoverCodeFiles(t *testing.T) { t.Fatalf("Expected %d files, got %d: %v", len(expected), len(discovered), discovered) } - // Order might vary depending on OS, but WalkDir is usually deterministic. - // We sort just in case or do a map check. found := make(map[string]bool) for _, f := range discovered { found[f] = true @@ -131,3 +186,10 @@ func TestDiscoverCodeFiles(t *testing.T) { } } } + +func TestVerifyGitRepo(t *testing.T) { + nonGitDir := t.TempDir() + if err := tools.VerifyGitRepo(nonGitDir); err == nil { + t.Fatal("Expected error for non-git directory, got nil") + } +} diff --git a/internal/tools/git_tools.go b/internal/tools/git_tools.go index 9f6ff1f..a20a267 100644 --- a/internal/tools/git_tools.go +++ b/internal/tools/git_tools.go @@ -7,28 +7,15 @@ import ( "strings" ) -// RunGit executes a git command in the specified repo directory. -func RunGit(repoRoot string, args ...string) (string, error) { - cmd := exec.Command("git", append([]string{"--no-pager"}, args...)...) +// VerifyGitRepo checks if git is available and the directory is a git repository. +func VerifyGitRepo(repoRoot string) error { + cmd := exec.Command("git", "--no-pager", "rev-parse", "--is-inside-work-tree") cmd.Dir = repoRoot - var stdout, stderr bytes.Buffer - cmd.Stdout = &stdout + var stderr bytes.Buffer cmd.Stderr = &stderr - err := cmd.Run() - trimmedOut := strings.TrimSpace(stdout.String()) - if err != nil { - trimmedErr := strings.TrimSpace(stderr.String()) - return trimmedOut, fmt.Errorf("git command failed: %v, stderr: %s", err, trimmedErr) - } - return trimmedOut, nil -} - -// VerifyGitRepo checks if git is available and the directory is a git repository. -func VerifyGitRepo(repoRoot string) error { - _, err := RunGit(repoRoot, "rev-parse", "--is-inside-work-tree") - if err != nil { - return fmt.Errorf("not a git repository (or any of the parent directories): %w", err) + if err := cmd.Run(); err != nil { + return fmt.Errorf("not a git repository (or any of the parent directories): %w, stderr: %s", err, strings.TrimSpace(stderr.String())) } return nil } diff --git a/wiki/.metadata.json b/wiki/.metadata.json index 4224127..1a29fcb 100644 --- a/wiki/.metadata.json +++ b/wiki/.metadata.json @@ -1,143 +1,83 @@ { "version": 1, - "steps_hash": "61d594e4ff13def120bfbc4dbb8f7556458b5da07bca0af9ff454967f8afc215", + "steps_hash": "f2dbf76e88087127bf3041564238fff6ee4695b81ab77ad6e46769a6958db2d3", "files": { - "cmd/cmd_test.go": { - "sha256": "028791d1a295f037bfbe579da61292cb99c3f25a61c3d8520eee233831c1deb8", - "facts": "#### [API_SIGNATURES]\n* TestRootFlags(t *testing.T)\n* TestCheckInitStatus(t *testing.T)\n* TestPromptString(t *testing.T)\n* TestPromptStringList(t *testing.T)\n* TestPromptContextSize(t *testing.T)\n* TestPromptIgnores(t *testing.T)\n* TestLoadInitialSetupConfig(t *testing.T)\n* TestRunSetupFlow(t *testing.T)\n\n#### [BUSINESS_LOGIC]\n**Primary Domain Problem**\nThe code manages the command-line interface (CLI) and interactive configuration lifecycle for project setup. It ensures that project initialization and update operations are performed in the correct state (e.g., preventing re-initialization of existing projects) and handles the persistence of user configuration settings.\n\n**High-Level Algorithm Steps**\n1. **Parse CLI Arguments:** Extract configuration parameters from command-line flags.\n2. **Verify Project State:** Determine if a project is initialized by checking for the existence of specific directories and metadata files.\n3. **Validate Operation Mode:** \n * Allow initialization only if the project is uninitialized.\n * Allow updates only if the project is already initialized.\n4. **Capture User Input:** Collect configuration values through interactive prompts, employing fallback logic for empty or invalid inputs.\n5. **Manage Configuration Persistence:** Load or save configuration settings to the file system.\n\n#### [STATE_AND_CONCURRENCY]\n**Mutable State**\n* `modelIDFlag`: Global variable modified in `TestRootFlags`.\n* `numCtxFlag`: Global variable modified in `TestRootFlags`.\n* `RootCmd`: Global variable modified via `.SetArgs()` and `.Execute()` in `TestRootFlags`.\n\n**Synchronization Mechanisms**\n* None identified.\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n**External I/O**\n* **Disk**:\n * Directory creation via `t.TempDir()` and `os.MkdirAll`.\n * File creation and writing via `os.WriteFile` (specifically `.metadata.json`).\n * Configuration file persistence and retrieval via `config.SaveConfig` and `config.LoadConfig`.\n* **Input**:\n * Simulated user input via `bufio.Reader`.\n\n**Error Propagation**\n* **Error Return Codes**: The functions `checkInitStatus`, `promptString`, `promptStringList`, `promptContextSize`, `promptIgnores`, `runSetupFlowWithReader`, `config.SaveConfig`, and `config.LoadConfig` propagate errors via return values.\n* **Swallowed Errors**: The return value from `RootCmd.Execute()` is swallowed in `TestRootFlags`." - }, "cmd/init.go": { - "sha256": "995bf2f9da724383666ec4f58da72bdea798d316266511b84faae8447fdd4682", - "facts": "#### [API_SIGNATURES]\n# Public API Surface (cmd/init.go)\n\nThis file defines **no exported or public elements**. All identifiers follow Go's unexported naming conventions:\n\n| Element | Visibility | Notes |\n|---|---|---|\n| `initCmd` | Unexported (`var`) | Cobra command struct literal; lowercase |\n| `init()` | Unexported (`func`) | Registers the cobra command with `RootCmd`; references an external `RootCmd` not defined in this file |\n\n**Conclusion:** No public API surface is exposed by this file.\n\n#### [BUSINESS_LOGIC]\n## Primary Domain Problem\n\nThis code establishes a CLI entry point for initializing project documentation. It triggers an engine operation that scans a repository and generates initial wiki markdown pages. The command is wired into a root cobra command tree, meaning the application supports multi-step operations (init being one of them).\n\n## High-Level Algorithm Steps\n\n1. Register an `init` subcommand under the root CLI command\n2. On invocation, delegate to an engine operation identified by `ModeInit`\n3. The engine scans the repository structure and produces a first-pass set of wiki markdown files\n\n#### [STATE_AND_CONCURRENCY]\nNo mutable state\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n## External I/O Analysis: `cmd/init.go`\n\n**No direct external side effects visible in this file.** This file is a thin CLI command wrapper. All actual work is delegated to `internal/engine.ExecuteCommand(engine.ModeInit)`. Since that module's source was not provided, its internal interactions (network calls, disk writes, database queries, etc.) cannot be analyzed here — those must be inferred from the engine package separately.\n\n**Dependencies:**\n- `github.com/arrase/code-reducer/internal/engine` — local package; external I/O behavior unknown without its source.\n- `github.com/spf13/cobra` — CLI framework; no external I/O in this file.\n\n## Error Propagation\n\nThe error handling pattern is straightforward and follows cobra conventions:\n\n```go\nRunE: func(cmd *cobra.Command, args []string) error {\n if err := executeCommand(engine.ModeInit); err != nil {\n return err // propagate up to cobra's error handling\n }\n return nil // success path\n},\n```\n\n- If `executeCommand` returns a non-nil error, it is returned directly to the caller (cobra), which will typically print a usage message and exit with status code 1.\n- No wrapping, retry logic, or panic recovery is present — errors pass through as-is." + "sha256": "f8db328c7ca4388b92667ad2f9ef2a1ebf8c8f00d35088ebfb4306e357ed743e", + "facts": "#### [API_SIGNATURES]\n# Public API Surface: `cmd/init.go`\n\n## None\n\n**No exported or public elements are defined in this file.** All identifiers (`initCmd`, `init`, `executeCommand`) are lowercase (unexported). `RootCmd` is referenced but not defined in this file.\n\n#### [BUSINESS_LOGIC]\n**Primary Domain Problem:**\nThis code provides a CLI entry point that triggers an initial documentation build. When invoked, it scans the repository and generates the first set of wiki markdown pages.\n\n**High-Level Algorithm Steps:**\n1. Register a new command named \"init\" with the root command group\n2. On execution, delegate to a shared command runner with the `ModeInit` engine mode\n3. The engine mode handles the repository scan and wiki page generation\n\n**None**\n\n#### [STATE_AND_CONCURRENCY]\nGlobal variable `initCmd` (`*cobra.Command`) is declared at package scope and initialized once. It is never modified after initialization. No synchronization mechanisms are present.\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n**External I/O:**\n- No external I/O is directly performed in this file. The `RunE` handler delegates to `executeCommand(engine.ModeInit)`, but the behavior of `executeCommand` is not visible in this file.\n\n**Error Propagation:**\n- Errors from `executeCommand` are returned directly to the cobra command framework via the `RunE` return value. The cobra framework will then display the error to the user and exit with a non-zero status code.\n- No panic or crash paths are present in this file." }, "cmd/root.go": { - "sha256": "d20db30dc9ae4b7208aecf6ccab88be4d36a2b4336adda7cbd6898a806923558", - "facts": "#### [API_SIGNATURES]\n# Public API Surface: `cmd` package (root.go)\n\n## Variables\n\n| Element | Type | Description |\n|---|---|---|\n| `RootCmd` | `*cobra.Command` | Root cobra command instance used as CLI entry point. |\n\n## Functions\n\n### Package-level functions\n\n| Function | Parameters | Return Type | Description |\n|---|---|---|---|\n| `init()` | — | — | Registers persistent flags (`--model-id`, `--num-ctx`) on `RootCmd`. |\n| `executeCommand` | `mode engine.Mode` | `error` | Validates git repo, configures setup if needed, resolves config, and runs the engine. |\n| `checkAndRunSetup` | `repoRoot string` | `error` | Checks whether a config file exists; if not and stdin is a TTY, invokes `RunSetupFlow`. |\n| `checkInitStatus` | `repoRoot, docsDir string`, `mode engine.Mode` | `error` | Validates init state based on the requested mode (`engine.ModeInit` or `engine.ModeUpdate`). |\n| `runEngine` | `repoRoot string`, `cfg *config.Config`, `mode engine.Mode` | `error` | Sets up signal handling, creates an engine runner, and executes documentation generation. |\n\n## Notes\n\n- All package-level functions in this file use lowercase identifiers (unexported). Only `init()` is a standard Go initialization function; the others are internal implementation details.\n- The only exported public element from this file is `RootCmd`.\n\n#### [BUSINESS_LOGIC]\n**Core Purpose:** \nA CLI tool that drives an LLM-based documentation agent through three lifecycle modes: `init`, `update`, and one additional mode (likely `run`). It manages project state transitions—configuring a setup phase when needed, validating git repository requirements, and executing the engine while streaming status/error events to stdout/stderr.\n\n**High-Level Algorithm Steps:**\n\n1. **Resolve working directory** — capture current path for all subsequent relative operations.\n2. **Verify Git repo** — fail early if the project is not a git repository.\n3. **Auto-run setup (if needed)** — when no prior configuration file exists and stdin is a terminal, trigger the `setup` flow; otherwise report an error.\n4. **Resolve configuration** — load or construct config from disk + CLI flags (`--model-id`, `--num-ctx`).\n5. **Check init status against mode** — enforce that:\n - `init` mode fails if already initialized (suggest `update`),\n - `update` mode fails if not yet initialized (suggest `init`).\n6. **Execute engine with signal handling** — register interrupt/terminate handlers, instantiate the runner, and stream events of three types (`status`, `error`, other) to stdout or stderr as appropriate.\n\n#### [STATE_AND_CONCURRENCY]\n**Global Variables (Mutable State):**\n\n| Variable | Type | Mutation Location | Notes |\n|---|---|---|---|\n| `modelIDFlag` | string | `init()` via `StringVar` | Set once at startup. Read-only after that within this file. |\n| `numCtxFlag` | string | `init()` via `StringVar` | Set once at startup. Read-only after that within this file. |\n\n**Synchronization Mechanisms:** None observed in this file. No locks, mutexes, atomics, channels, or goroutine patterns visible here.\n\n**Shared State:** The two global flags (`modelIDFlag`, `numCtxFlag`) are shared across all calls to `executeCommand` and any function that reads them (e.g., `checkAndRunSetup`, `runEngine`). They persist for the lifetime of the process.\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n## External I/O Interactions\n\n### Disk / Filesystem\n| Operation | Location | Purpose |\n|-----------|----------|---------|\n| `os.Getwd()` | `executeCommand` | Reads current working directory path |\n| `tools.VerifyGitRepo(repoRoot)` | `executeCommand` | Validates git repository presence (reads `.git` or equivalent markers) |\n| `config.ConfigExists(repoRoot)` | `checkAndRunSetup` | Checks whether config file exists on disk |\n| `RunSetupFlow(repoRoot)` | `checkAndRunSetup` | Writes configuration files and project structure to disk during setup |\n| `engine.IsInitialized(repoRoot, docsDir)` | `checkInitStatus` | Reads filesystem state (likely checks for init marker files in docs directory) |\n\n### Stdout / Stderr\n| Output Channel | Condition | Content |\n|---------------|-----------|---------|\n| stdout (`fmt.Println`) | `ev.Type == engine.EventStatus` OR unknown event type | Status messages and generic events from the runner |\n| stderr (`os.Stderr`) | `ev.Type == engine.EventError` | Error messages prefixed with \"Error: \" |\n\n### Signal Handling\n- Registers `os.Interrupt` (SIGINT) and `syscall.SIGTERM` via `signal.NotifyContext(context.Background(), ...)` to enable graceful cancellation of the runner. The context is stored in a local variable (`ctx`) and deferred stop on exit.\n\n---\n\n## Error Propagation\n\n### Wrapped Errors (preserved chain via `%w`)\n- **Working directory failure**: Wraps `os.Getwd()` error as `\"failed to get current working directory: %w\"` — caller can use `errors.Is`/`errors.As`.\n- **Setup flow failure**: Returns error from `RunSetupFlow(repoRoot)` directly without wrapping — caller sees original error.\n- **Runner failure**: Wraps any error from `runner.Run(...)` as `\"documentation run failed: %w\"`.\n\n### Pass-Through Errors (no wrapping)\n- **Git verification**: Error returned by `tools.VerifyGitRepo` is propagated verbatim to the parent call site.\n- **Config resolution**: Error from `config.ResolveConfig` is passed through unchanged.\n- **Init status checks**: All errors are *new* `fmt.Errorf(...)` messages — no wrapping of previous errors, so error chain terminates at this point.\n\n### User-Facing Error Messages (created inline)\nAll generated in `checkAndRunSetup` and `checkInitStatus`:\n- Config file missing on non-TTY: `\"configuration file %s does not exist... Please run 'code-reducer setup'\"`\n- Init already done: `\"the project has already been initialized. Please use 'code-reducer update'\"`\n- Not yet initialized: `\"the project has not been initialized yet. Please run 'code-reducer init' first\"`\n\n### No Panic / Crash Path\nNo `panic()` calls or unhandled recover blocks are present in this file. All errors are handled via return values and propagated upward to the CLI entry point." + "sha256": "4b2b0a19036f1f6bbafafc3db7a2200d399f329bedd7411b94d974443392e4ea", + "facts": "#### [API_SIGNATURES]\n## Public API Surface: `cmd/root.go`\n\n### Exported Variables\n\n| Element | Type | Description |\n|---------|------|-------------|\n| `RootCmd` | `*cobra.Command` | Root cobra command for the CLI application |\n\n### Unexported Functions\n\n| Function | Parameters | Return |\n|----------|-----------|--------|\n| `init()` | (none) | (none) |\n| `executeCommand(mode engine.RunMode)` | `mode engine.RunMode` | `error` |\n| `checkAndRunSetup(repoRoot string)` | `repoRoot string` | `error` |\n| `checkInitStatus(repoRoot, docsDir string, mode engine.RunMode)` | `repoRoot string`, `docsDir string`, `mode engine.RunMode` | `error` |\n| `runEngine(repoRoot string, cfg *config.Config, mode engine.RunMode)` | `repoRoot string`, `cfg *config.Config`, `mode engine.RunMode` | `error` |\n\n### Notes\n\n- `modelIDFlag` and `numCtxFlag` are unexported `string` variables bound to persistent flags on `RootCmd`.\n- `engine.RunMode` and `config.Config` are referenced types defined in other packages.\n\n#### [BUSINESS_LOGIC]\n**Domain Problem:**\nA CLI tool that orchestrates a documentation agent to write and maintain project wikis. It validates the execution environment, ensures configuration is present, and dispatches the documentation engine based on the requested operation mode.\n\n**High-Level Algorithm Steps:**\n\n1. Resolve the current working directory.\n2. Verify the directory is a git repository.\n3. If no configuration file exists and stdin is a terminal, trigger the setup flow (otherwise error out).\n4. Resolve the configuration by merging CLI flags with any existing config file.\n5. Validate that the requested mode (init or update) is consistent with the project's initialization state.\n6. Execute the documentation engine with signal handling for graceful shutdown.\n\n#### [STATE_AND_CONCURRENCY]\nNo mutable state\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n**External I/O Interactions:**\n\n- **Disk I/O:** `os.Getwd()` reads the current working directory. `tools.VerifyGitRepo(repoRoot)` checks for a `.git` directory. `config.ResolveConfig(repoRoot, ...)` reads the configuration file from disk. `engine.IsInitialized(repoRoot, docsDir)` checks if the docs directory exists.\n- **Network I/O:** `engine.Run(ctx, repoRoot, cfg, mode, ...)` invokes the engine, which performs LLM calls (network I/O to an external model provider).\n- **Signal Handling:** `signal.NotifyContext` registers handlers for `os.Interrupt` and `syscall.SIGTERM` to allow graceful shutdown.\n- **Terminal Detection:** `isatty.IsTerminal(os.Stdin.Fd())` and `isatty.IsCygwinTerminal(os.Stdin.Fd())` check if stdin is a terminal.\n- **Stdout:** `fmt.Println(msg)` writes progress messages to stdout.\n\n**Error Propagation:**\n\n- All functions in `root.go` return `error` type. Errors propagate up the call chain via return values.\n- `executeCommand` returns errors from `os.Getwd()`, `tools.VerifyGitRepo`, `checkAndRunSetup`, `config.ResolveConfig`, `checkInitStatus`, and `runEngine`.\n- `runEngine` wraps engine errors with `fmt.Errorf(\"documentation run failed: %w\", err)`.\n- `checkAndRunSetup` returns errors when the config file is missing and stdin is not a terminal.\n- `checkInitStatus` returns errors when init/update mode conflicts with current state.\n- No panics are used. No exceptions. Errors are swallowed only when returned to the caller (not handled within this file)." }, "cmd/setup.go": { - "sha256": "b93775150eb0ded835e35052f399dfd840a79469a35e3ba2741b3c1fd3149038", - "facts": "#### [API_SIGNATURES]\n* `RunSetupFlow(repoRoot string) error`\n\n#### [BUSINESS_LOGIC]\n### Primary Domain Problem\nThe code facilitates interactive configuration management for a tool that interfaces with Large Language Models (LLMs) to process codebases. It provides a mechanism to define and persist environment-specific settings, including LLM connection details, processing constraints, file exclusion rules, and directory paths.\n\n### High-level Algorithm Steps\n1. **Identify Environment:** Determine the current working directory to locate configuration files.\n2. **Initialize Configuration:** Load existing configuration settings from the project root and merge them with predefined default values where settings are missing or empty.\n3. **Interactive Input Collection:** Iteratively prompt the user for specific parameters, allowing the user to accept current values by providing no input:\n * LLM Model ID.\n * Ollama Base URL.\n * Ollama Context Size.\n * File/directory ignore patterns.\n * Documentation directory path.\n4. **Consolidate Settings:** Assemble a complete configuration object using both the user-provided inputs and the existing/default values for fields not modified by the user.\n5. **Persist Configuration:** Save the final configuration object to a file in the repository root.\n\n#### [STATE_AND_CONCURRENCY]\n**Mutable State**\n* `setupCmd` (package-level `*cobra.Command`)\n* `RootCmd` (modified via `AddCommand` in `init`)\n\n**Synchronization Mechanisms**\n* None\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n**External I/O**\n* **Disk**:\n * `os.Getwd`: Retrieves the current working directory.\n * `config.LoadConfig`: Loads configuration from `repoRoot`.\n * `config.SaveConfig`: Writes configuration to `repoRoot`.\n* **Standard Input**:\n * `os.Stdin` (via `bufio.Reader`): Used to capture user input for Model ID, Ollama Base URL, Context Size, ignore patterns (files/directories), and the documentation directory.\n* **Standard Output**:\n * `fmt.Println` and `fmt.Printf`: Used for status messages, interactive prompts, and success notifications.\n\n**Error Propagation**\n* **`os.Getwd` error**: Wrapped with context and returned.\n* **`config.LoadConfig` error**: Swallowed; returns a default `config.Config` if an error occurs or if the returned config is `nil`.\n* **`strconv.Atoi` error (in `promptContextSize`)**: Swallowed; returns the `defaultVal`.\n* **`reader.ReadString` error**: Wrapped and returned via `promptString`, `promptStringList`, `promptContextSize`, and `promptIgnores`.\n* **`config.SaveConfig` error**: Wrapped and returned." + "sha256": "c53f212f7f2450c80ab69e2188530f58f6e4025781e0d05894f3d1753c5377aa", + "facts": "#### [API_SIGNATURES]\n## Public API Surface\n\n### Functions\n\n| Function | Parameters | Return Type |\n|---|---|---|\n| `RunSetupFlow` | `repoRoot string` | `error` |\n\n### Variables\n\n| Variable | Type |\n|---|---|\n| `setupCmd` | `*cobra.Command` |\n\n### Types\n\nNone\n\n#### [BUSINESS_LOGIC]\n**Primary Domain Problem:**\nInteractive CLI setup wizard that generates or updates a `.code-reducer.yaml` configuration file, collecting LLM model settings, base URL, context size, ignore patterns, and documentation directory from the user.\n\n**High-Level Algorithm Steps:**\n1. Load any existing configuration from the current working directory (fall back to empty config if not found)\n2. Apply default values to the loaded configuration\n3. Prompt the user sequentially for each configuration value:\n - LLM Model ID\n - Ollama Base URL\n - Ollama Context Size (with numeric validation and default fallback)\n - Ignore patterns (comma-separated list, supports \"clear\"/\"none\" to reset)\n - Documentation directory\n4. Persist the resulting configuration to the local config file\n\n#### [STATE_AND_CONCURRENCY]\nGlobal variables:\n- `setupCmd` (package-level `*cobra.Command`) — initialized once in `init()`, never modified after creation.\n\nShared state / synchronization:\n- None. No mutexes, locks, atomic types, or async patterns are present.\n\nMutable state summary:\n- `setupCmd` is immutable after init.\n- `current` in `runSetupFlowWithReader` is a local pointer modified in place, but scoped to the function.\n- No thread safety mechanisms are used.\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n**External I/O Interactions:**\n\n- **Disk Read:** `config.LoadConfig(repoRoot)` reads the configuration file from the repository root directory.\n- **Disk Write:** `config.SaveConfig(repoRoot, current)` writes the updated configuration to the repository root directory.\n- **Stdin Read:** `bufio.NewReader(os.Stdin)` creates a reader for standard input. All user prompts (`promptString`, `promptContextSize`, `promptIgnores`) read from stdin via `reader.ReadString('\\n')`.\n- **Stdout Write:** `fmt.Println` and `fmt.Printf` write setup messages and success confirmation to stdout.\n- **Filesystem State:** `os.Getwd()` reads the current working directory path.\n\n**Error Propagation:**\n\n- Errors from `os.Getwd()` are wrapped with `fmt.Errorf` and returned to the caller.\n- Errors from `config.LoadConfig` are caught; if non-nil, an empty `config.Config{}` is used instead.\n- Errors from `promptString`, `promptContextSize`, `promptIgnores`, and `config.SaveConfig` are wrapped with `fmt.Errorf` and returned up the call chain.\n- No panics or crashes are present. All errors propagate as return values to the top-level `RunSetupFlow` function.\n- No error logging occurs; errors are only returned to the caller." }, "cmd/update.go": { - "sha256": "56faface0ab0f0c994482e1c88b145dffec088858bd445dc56a63df977f4b638", - "facts": "#### [API_SIGNATURES]\n## Public API Surface — `cmd/update.go`\n\n**Note:** This file contains no exported (uppercase) identifiers. All public-facing functionality is wired through the unexported `updateCmd` variable and auto-called `init()` function, which are implementation details of the private `cmd` package.\n\n### No exported elements found\n\n#### [BUSINESS_LOGIC]\n## Core Purpose\n\nThis code implements the `update` subcommand for a CLI tool that manages project documentation. It triggers an incremental update of wiki pages by scanning files changed since the last documented commit and applying those changes to existing wiki content.\n\n## High-Level Algorithm Steps\n\n1. Register a new cobra command named `update` under the root command\n2. On execution, delegate to the internal engine with mode `ModeUpdate`\n3. The engine scans for modified files compared against the last committed documentation baseline\n4. Updated content is merged back into existing wiki pages\n\n#### [STATE_AND_CONCURRENCY]\nNo mutable state\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n**External Side Effects:**\n\n- The RunE handler in `updateCmd` is a pure delegation wrapper. It calls `executeCommand(engine.ModeUpdate)` and returns whatever error that call produces. All actual I/O (network, disk, database) happens inside the unshown `engine` package — this file itself performs none.\n- No explicit network calls, no file reads/writes, no database queries are visible in this file's source.\n\n**Error Propagation:**\n\n- Errors propagate upward through the Cobra command chain via the standard `RunE func(cmd *cobra.Command, args []string) error` signature:\n - If `executeCommand(engine.ModeUpdate)` returns an `error`, it is returned directly to the caller (the CLI invocation path).\n - If it returns `nil`, the function returns `nil` — Cobra treats this as a successful execution.\n- No panic, no silent swallow. Any error from the engine's update logic surfaces at the top level of the CLI.\n\n**Summary:** This file is a thin command-definition shell; all external behavior and error handling live in the unreviewed `internal/engine` package." + "sha256": "e776a2d66e5d050782b6c8c9015d2d668b8cf0e95a267e2ef6f536e039e8230b", + "facts": "#### [BUSINESS_LOGIC]\n**Primary Domain Problem:**\nIncrementally update documentation (wiki pages) for files that have changed since the last documented commit, using cached SHA-256 hashes to identify modifications.\n\n**High-Level Algorithm Steps:**\n1. Scan the repository to identify files modified since the last documented commit\n2. Use cached SHA-256 hashes to determine which files have changed\n3. Update the corresponding wiki pages for the identified changed files\n\n**Domain Rules:**\n- Updates are incremental (only changed files are processed)\n- SHA-256 hashes are cached for change detection\n- The operation is tied to the last documented commit as the baseline\n\n#### [STATE_AND_CONCURRENCY]\nGlobal variables:\n- `updateCmd` — package-level `*cobra.Command` assigned once; never modified after initialization.\n\nShared state / synchronization:\n- None. No mutexes, locks, atomic types, or async mechanisms are present.\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n**External I/O:** None (directly in this file). The actual I/O logic is delegated to `internal/engine` package via `executeCommand(engine.ModeUpdate)`. No file operations, network calls, or database interactions are visible here.\n\n**Error Propagation:** Errors from `executeCommand` are returned directly to cobra's command framework via `RunE`. The error propagates up through the cobra command chain to the CLI entry point. No panic or crash handling is implemented in this file." }, "internal/config/config.go": { - "sha256": "a89a8e4d3499605f8b070f9169c91543bbdc7ad7cf8e60e1fbd46b036da09917", - "facts": "#### [API_SIGNATURES]\n# Public API Surface of `internal/config` Package\n\n## Constants\n\n| Name | Type | Value / Description |\n|---|---|---|\n| `CodeReducerModelIDEnvKey` | `string` | `\"CODE_REDUCER_MODEL_ID\"` — env key for model ID override. |\n| `OllamaBaseURLEnvKey` | `string` | `\"OLLAMA_BASE_URL\"` — env key for Ollama URL override. |\n| `OllamaNumCtxEnvKey` | `string` | `\"OLLAMA_NUM_CTX\"` — env key for context size override. |\n| `OllamaDefaultBaseURL` | `string` | `\"http://localhost:11434\"` — default URL for local Ollama API. |\n| `OllamaDefaultModelID` | `string` | `\"ornith:9b\"` — default LLM model ID. |\n| `OllamaDefaultNumCtx` | `int` | `8192` — default context size. |\n| `DefaultDocsDir` | `string` | `\"wiki\"` — default documentation folder name. |\n| `ConfigFileName` | `string` | `\".code-reducer.yaml\"` — configuration filename. |\n\n## Types\n\n### `ExtractionStep`\n\n```go\ntype ExtractionStep struct {\n Name string // yaml:\"name\"\n Prompt string // yaml:\"prompt\"\n}\n```\n\n### `Config`\n\n```go\ntype Config struct {\n ModelID string // yaml:\"model_id\"\n OllamaBaseURL string // yaml:\"ollama_base_url\"\n OllamaNumCtx int // yaml:\"ollama_num_ctx\"\n DocsDir string // yaml:\"docs_dir\"\n SystemPrompt string // yaml:\"system_prompt\"\n ModuleSynthesisPrompt string // yaml:\"module_synthesis_prompt\"\n ArchitecturePrompt string // yaml:\"architecture_prompt\"\n FileFactConsolidationPrompt string // yaml:\"file_fact_consolidation_prompt\"\n ExtractionSteps []ExtractionStep // yaml:\"extraction_steps\"\n Ignore []string // yaml:\"ignore\"\n}\n```\n\n## Variables\n\n### `DefaultExtractionSteps`\n\n- **Type:** `[]ExtractionStep`\n- **Value:** Pre-populated slice containing four default extraction steps:\n - `{Name: \"API_SIGNATURES\", ...}` — extracts public API surface.\n - `{Name: \"BUSINESS_LOGIC\", ...}` — explains primary domain problem and high-level algorithm steps.\n - `{Name: \"STATE_AND_CONCURRENCY\", ...}` — identifies mutable state, shared states, synchronization mechanisms; outputs `\"No mutable state\"` if entirely stateless.\n - `{Name: \"ERRORS_AND_SIDE_EFFECTS\", ...}` — details external I/O interactions and error propagation; outputs `\"No external side effects\"` if no I/O exists.\n\n#### [BUSINESS_LOGIC]\n**Primary Domain Problem:**\n\nThis code defines the configuration schema for an AI-powered static analysis tool that produces technical documentation from source code. It configures how a local LLM (via Ollama) parses source files through multiple extraction phases to synthesize structured docs.\n\n**High-Level Algorithm Steps:**\n\n1. **API Surface Extraction** — Extract public types, functions, methods, and their signatures without explaining internal logic.\n2. **Business Logic Synthesis** — Explain the primary domain problem solved by the code and list high-level algorithm steps, ignoring implementation syntax.\n3. **State \u0026 Concurrency Mapping** — Identify mutable global/state variables and synchronization mechanisms; report \"No mutable state\" if none exist.\n4. **Error \u0026 Side Effect Analysis** — Detail interactions with external systems (network, disk, databases) and how errors propagate or are swallowed.\n\nThese four extraction steps feed into a multi-step LLM pipeline that ultimately produces module-level documentation and architecture-wide synthesis documents.\n\n#### [STATE_AND_CONCURRENCY]\n**Mutable State:** `DefaultExtractionSteps` — declared as a package-level `var` initialized with a slice of four `ExtractionStep` structs. No synchronization mechanism (mutex, atomic type, etc.) protects it in this file.\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n**External I/O:** No external side effects. This file contains only type definitions (`ExtractionStep`, `Config`), constants, default values, and environment variable key names. No functions perform reads, writes, network calls, or any system interactions.\n\n**Error Propagation:** N/A — no error-returning functions exist in this file." - }, - "internal/config/config_test.go": { - "sha256": "9373e2c7629b6b302ed73c20dac13bd8a79b979c4d71bf7df4854ed3a8a5b804", - "facts": "#### [API_SIGNATURES]\n* `LoadConfig(dir string) (*Config, error)`\n* `SaveConfig(dir string, cfg *Config) error`\n* `ResolveConfig(dir string, modelID string, numCtx string) (*Config, error)`\n* `Config` (struct)\n * `ModelID string`\n * `OllamaNumCtx int`\n * `OllamaBaseURL string`\n* `ConfigFileName`\n* `OllamaNumCtxEnvKey`\n* `CodeReducerModelIDEnvKey`\n* `OllamaBaseURLEnvKey`\n\n#### [BUSINESS_LOGIC]\n**Primary Domain Problem**\nManaging application configuration through a hierarchical resolution process that combines settings from local files, environment variables, and command-line flags.\n\n**High-Level Algorithm Steps**\n1. Load configuration data from a specified directory.\n2. Overlay configuration values sourced from environment variables.\n3. Overlay configuration values sourced from command-line flags.\n4. Validate resolved numerical parameters to ensure they are valid and meet specific non-zero requirements.\n\n#### [STATE_AND_CONCURRENCY]\nMutable state:\n- Environment variables (modified via `t.Setenv` using `OllamaNumCtxEnvKey`, `CodeReducerModelIDEnvKey`, and `OllamaBaseURLEnvKey`)\n- Filesystem (modified via `t.TempDir` and `os.WriteFile`)\n\nSynchronization mechanisms:\n- None\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n**External I/O**\n* **Disk**: `SaveConfig` writes configuration data to the filesystem. `LoadConfig` reads configuration data from the filesystem. `os.WriteFile` writes data to the filesystem.\n* **Environment**: `ResolveConfig` accesses environment variables to retrieve configuration values.\n\n**Error Propagation**\n* **Return Values**: `LoadConfig`, `SaveConfig`, and `ResolveConfig` propagate errors to the caller via return values.\n* **Error Conditions**:\n * `LoadConfig` returns an error if a file is not found or contains invalid YAML.\n * `SaveConfig` returns an error during write operations.\n * `ResolveConfig` returns an error if environment variables or provided arguments contain invalid or non-numeric values." + "sha256": "d4f6d3b6fbb3dc0933991b99baa1c6b818dfce6be3e63a17e64130d8e58172e5", + "facts": "#### [API_SIGNATURES]\n- `CodeReducerModelIDEnvKey` = \"CODE_REDUCER_MODEL_ID\"\n- `OllamaBaseURLEnvKey` = \"OLLAMA_BASE_URL\"\n- `OllamaNumCtxEnvKey` = \"OLLAMA_NUM_CTX\"\n- `OllamaDefaultBaseURL` = \"http://localhost:11434\"\n- `OllamaDefaultModelID` = \"ornith:9b\"\n- `OllamaDefaultNumCtx` = 8192\n- `DefaultDocsDir` = \"wiki\"\n- `ConfigFileName` = \".code-reducer.yaml\"\n- `DefaultSystemPrompt` = multi-line string defining Code-Reducer role and rules (extract facts, no speculation, no code blocks)\n- `DefaultModuleSynthesisPrompt` = multi-line string for folder summary generation\n- `DefaultArchitecturePrompt` = multi-line string for global architecture document generation\n- `DefaultQuickstartPrompt` = multi-line string for developer quickstart guide generation\n- `DefaultFileFactConsolidationPrompt` = multi-line string for fact consolidation and deduplication\n- `ExtractionStep` struct: `Name` string, `Prompt` string\n- `Config` struct: `ModelID` string, `OllamaBaseURL` string, `OllamaNumCtx` int, `DocsDir` string, `SystemPrompt` string, `ModuleSynthesisPrompt` string, `ArchitecturePrompt` string, `QuickstartPrompt` string, `FileFactConsolidationPrompt` string, `ExtractionSteps` []ExtractionStep, `Ignore` []string\n- `DefaultExtractionSteps` = []ExtractionStep containing API_SIGNATURES, BUSINESS_LOGIC, STATE_AND_CONCURRENCY, ERRORS_AND_SIDE_EFFECTS\n\n#### [BUSINESS_LOGIC]\n**BUSINESS_LOGIC:**\n\n- The code defines a configuration package for a tool called Code-Reducer, which is a technical code analyzer and documentation writer.\n- It provides a configuration schema (Config struct) that supports model ID override, Ollama base URL, context size, documentation directory, system prompts, module synthesis prompts, architecture prompts, quickstart prompts, file fact consolidation prompts, extraction steps, and an ignore list.\n- It defines four default extraction steps: API_SIGNATURES (extract public API surface), BUSINESS_LOGIC (extract core purpose and domain rules), STATE_AND_CONCURRENCY (identify mutable state and thread safety), and ERRORS_AND_SIDE_EFFECTS (analyze external I/O and error propagation).\n- It provides default prompts for system instructions, module synthesis, architecture synthesis, quickstart synthesis, and file fact consolidation.\n- The configuration file is named `.code-reducer.yaml` and uses 0600 file permissions.\n- Default values are set for Ollama base URL (localhost:11434), default model (ornith:9b), default context size (8192), and default docs directory (wiki).\n\n#### [ERRORS_AND_SIDE_EFFECTS]\nBased on the provided source code, here are the verifiable facts about external I/O and error propagation:\n\n**File System Interactions:**\n- Configuration file `.code-reducer.yaml` is referenced (ConfigFileName constant)\n- File permission is set to 0600 (configFilePerm constant)\n\n**Network Interactions:**\n- Ollama API endpoint at `http://localhost:11434` (OllamaDefaultBaseURL constant)\n- Environment variable `OLLAMA_BASE_URL` allows overriding the base URL\n\n**Environment Variable Interactions:**\n- `CODE_REDUCER_MODEL_ID` - model ID override\n- `OLLAMA_BASE_URL` - Ollama URL override \n- `OLLAMA_NUM_CTX` - context size override\n\n**Error Handling:**\n- No explicit error handling patterns are shown in the provided config.go file\n- The configuration schema includes fields for model ID, base URL, context size, and various prompts, but error propagation logic is not visible in this file\n\n**Note:** The actual implementation details for how these interactions are performed (reading config files, making network calls, etc.) would be in the `internal/config/io.go` and `internal/config/resolve.go` files, which are not shown in the provided code." }, "internal/config/io.go": { - "sha256": "d979275e88e5ef59442f2ecf44417479d4fc3b4c0d4a75632afd0ddb306b5057", - "facts": "#### [API_SIGNATURES]\n# Public API Surface: `internal/config/io.go`\n\n## Functions\n\n### ConfigExists\n- **Package**: `config`\n- **Signature**: `func ConfigExists(cwd string) bool`\n- **Parameters**:\n - `cwd` — directory path (string)\n- **Returns**:\n - `bool` — true if `.code-reducer.yaml` exists in the specified directory\n\n### LoadConfig\n- **Package**: `config`\n- **Signature**: `func LoadConfig(cwd string) (*Config, error)`\n- **Parameters**:\n - `cwd` — directory path (string)\n- **Returns**:\n - `*Config` — parsed configuration pointer\n - `error` — non-nil if file I/O or YAML parsing fails\n\n### SaveConfig\n- **Package**: `config`\n- **Signature**: `func SaveConfig(cwd string, cfg *Config) error`\n- **Parameters**:\n - `cwd` — directory path (string)\n - `cfg` — configuration pointer to persist\n- **Returns**:\n - `error` — non-nil if write/rename/chmod operations fail\n\n---\n\n## Non-Exported Functions (for reference only; not part of public API)\n\n| Function | Signature |\n|----------|-----------|\n| `getConfigPath` | `func getConfigPath(cwd string) string` |\n| `formatYAML` | `func formatYAML(data []byte) string` |\n\n#### [BUSINESS_LOGIC]\n## Core Purpose\n\nConfiguration management for the **Code-Reducer** system — an AI-driven code synthesis tool that loads, persists, and validates user-configurable parameters (prompts, extraction steps, ignore lists) stored in `.code-reducer.yaml`.\n\n## High-Level Algorithm Steps\n\n1. **Resolve config path** — Build absolute filesystem path from current working directory + configured filename (`ConfigFileName`).\n2. **Check existence** — Stat the resolved path to determine whether a prior config exists at that location.\n3. **Load config** — Read file bytes, unmarshal YAML into a typed `Config` struct; return error if read or parse fails.\n4. **Format/configure output** — Normalize serialized YAML by inserting double newlines before specific prompt keys (`system_prompt`, `module_synthesis_prompt`, etc.) for consistent formatting.\n5. **Save config (atomic write)**:\n - Marshal the typed struct to YAML bytes.\n - Apply format normalization.\n - Write to a temporary file in the same directory.\n - Sync, close, chmod the temp file.\n - Atomically rename the temp file over the target path via `os.Rename`.\n\n#### [STATE_AND_CONCURRENCY]\nNo mutable state\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n## External I/O Analysis\n\n### Disk Operations (File System)\n\n| Function | Operation | Description |\n|---|---|---|\n| `ConfigExists` | `os.Stat()` | Reads directory metadata to check if `.code-reducer.yaml` exists. Returns `bool`, not error — caller must infer absence from `false`. |\n| `LoadConfig` | `os.ReadFile()` | Reads the entire config file into memory for parsing. |\n| `SaveConfig` | `os.CreateTemp()` + `os.Write()` + `os.Sync()` + `os.Chmod()` + `os.Rename()` | Writes config via atomic rename: creates temp file, writes content, syncs to disk, closes file descriptor, sets permissions (`fileMode`), then atomically renames temp over the target path. This is a standard write-through pattern. |\n\n### Library API Calls (Non-Standard)\n\n| Function | Dependency | Description |\n|---|---|---|\n| `LoadConfig` | `gopkg.in/yaml.v3` (`yaml.Unmarshal`) | Parses YAML config from byte slice into struct. Wraps parse errors with a custom prefix string. |\n| `SaveConfig` | `gopkg.in/yaml.v3` (`yaml.Marshal`) | Serializes struct back to YAML bytes before writing to disk. |\n\n### Error Propagation\n\n- **`LoadConfig`**: Returns `(nil, err)` on any failure — either from file read or YAML parse. The parsed error is wrapped via `fmt.Errorf(\"failed to parse yaml config: %w\", err)`.\n- **`SaveConfig`**: Returns `error` after each step fails. Errors are wrapped similarly with descriptive prefixes (`\"failed to create temp file:\"`, `\"failed to write config to temp file:\"`, etc.). The final operation is `os.Rename()`; if that fails, it returns the raw error without wrapping.\n- **`ConfigExists`**: Swallows all errors — returns `false` whether the file doesn't exist, the path is inaccessible due to permissions, or any other OS-level failure occurs. This is a silent swallow with no way for the caller to distinguish causes.\n\n### Potential Panic Points (Unhandled)\n\nThe deferred function in `SaveConfig`:\n```go\ndefer func() {\n tmpFile.Close()\n os.Remove(tmpName)\n}()\n```\nIf any error path triggers a panic during execution, or if the deferred cleanup itself panics, it is **not** recovered. No `recover()` wrapping exists anywhere in this file.\n\n### No External Side Effects (Confirmed Absent)\n\n- No network calls, HTTP clients, database drivers, or external API interactions.\n- All I/O targets are local filesystem paths (`cwd` + `.code-reducer.yaml`)." + "sha256": "4e7a76b20a33316aecb7af0bf13cdfc41a02e02534f6e4b7d9fe737b5b562466", + "facts": "#### [API_SIGNATURES]\n## Public API Surface\n\n### Functions\n\n| Function | Parameters | Return Types |\n|---|---|---|\n| `ConfigExists` | `cwd string` | `bool` |\n| `LoadConfig` | `cwd string` | `*Config`, `error` |\n| `SaveConfig` | `cwd string`, `cfg *Config` | `error` |\n\n### Types\n\n| Type | Description |\n|---|---|\n| `Config` | Referenced in `LoadConfig` and `SaveConfig` signatures; definition not present in this file |\n\n### Constants\n\n| Constant | Description |\n|---|---|\n| `ConfigFileName` | Referenced in `getConfigPath` and `SaveConfig`; definition not present in this file |\n| `configFilePerm` | Referenced in `SaveConfig`; definition not present in this file |\n\n### Unexported Functions (Internal Only)\n\n| Function | Parameters | Return Types |\n|---|---|---|\n| `getConfigPath` | `cwd string` | `string` |\n| `formatYAML` | `data []byte` | `string` |\n\n#### [BUSINESS_LOGIC]\n**Primary Domain Problem:**\nManages a `.code-reducer.yaml` configuration file for a code analysis tool. Handles the full lifecycle: locating the file, loading/parsing its contents, and persisting changes with atomic writes.\n\n**High-Level Algorithm Steps:**\n1. Resolve the config file path relative to the working directory\n2. Check whether the config file exists at that path\n3. Read the file and parse YAML into a structured `Config` object\n4. On save: marshal the config back to YAML, apply formatting (insert blank lines before specific keys), then write atomically to the same path\n\n#### [STATE_AND_CONCURRENCY]\nNo mutable state\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n**External I/O:**\n- **Disk reads:** `os.Stat` (in `ConfigExists`), `os.ReadFile` (in `LoadConfig`)\n- **Disk writes:** `tools.WriteFileAtomic` (in `SaveConfig`)\n- **No network, database, or API interactions** in this file.\n\n**Error Propagation:**\n- `LoadConfig`: returns `(*Config, error)`. If `os.ReadFile` fails, returns the raw error. If YAML parsing fails, wraps with `fmt.Errorf(\"failed to parse yaml config: %w\", err)`.\n- `SaveConfig`: returns `error`. If `yaml.Marshal` fails, wraps with `fmt.Errorf(\"failed to marshal yaml: %w\", err)`. Then passes the error through to `tools.WriteFileAtomic`.\n- `ConfigExists`: returns `bool` derived from `os.Stat` error check (`err == nil`).\n- `getConfigPath` and `formatYAML`: no error handling; errors are unhandled (panic on failure)." }, "internal/config/resolve.go": { - "sha256": "f560e11dfcac8b454621ecd0a57970fe788ed1b469a78bcdc99073f0d135c7a6", - "facts": "#### [API_SIGNATURES]\n* `ResolveConfig(repoRoot string, modelIDFlag string, numCtxFlag string) (*Config, error)`\n\n#### [BUSINESS_LOGIC]\n**Primary Domain Problem**\nThe code manages hierarchical configuration resolution, merging settings from multiple sources (CLI flags, environment variables, configuration files, and system defaults) into a single, validated configuration object.\n\n**High-Level Algorithm Steps**\n1. **Load Base Configuration:** Attempt to load the configuration file from the specified repository root.\n2. **Resolve Numerical Context:** Determine the context size by evaluating settings in this priority order: CLI flag \u003e Environment variable (with validation) \u003e Configuration file value (if $\u003e 0$) \u003e System default.\n3. **Resolve Model Identity:** Determine the model ID by evaluating settings in this priority order: CLI flag \u003e Environment variable \u003e Configuration file value \u003e System default.\n4. **Resolve Service Endpoints:** Determine the base URL by evaluating settings in this priority order: Environment variable \u003e Configuration file value \u003e System default.\n5. **Resolve Text Settings:** Select values for prompts and directory paths by preferring non-empty values from the configuration file over system defaults.\n6. **Finalize Lists and Defaults:** \n * Deduplicate the list of ignored items.\n * Use predefined default extraction steps if the configuration file provides an empty list.\n\n#### [STATE_AND_CONCURRENCY]\nNo mutable state\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n### External I/O\n* **Disk**: `loadBaseConfig` calls `LoadConfig` using `repoRoot` to attempt to read a configuration file.\n* **Environment Variables**: \n * `CodeReducerModelIDEnvKey` is read via `os.Getenv`.\n * `OllamaBaseURLEnvKey` is read via `os.Getenv`.\n * `OllamaNumCtxEnvKey` is read via `os.Getenv`.\n\n### Error Propagation\n* **`loadBaseConfig`**:\n * If `LoadConfig` returns an error that is not `os.ErrNotExist`, the error is wrapped and returned.\n * If `LoadConfig` returns `os.ErrNotExist`, the error is swallowed and an empty `Config` struct is returned.\n* **`resolveNumCtx`**:\n * If the environment variable `OllamaNumCtxEnvKey` is non-empty but fails `strconv.Atoi` or the resulting integer is $\\le 0$, an error is returned.\n * If `flagVal` is non-empty but fails `strconv.Atoi` or the resulting integer is $\\le 0$, an error is returned.\n* **`ResolveConfig`**:\n * Returns errors propagated from `loadBaseConfig` or `resolveNumCtx`." + "sha256": "37f0ca58f09a2522085222303d3f83bde3bc4829d265cdc6a697edbded661409", + "facts": "#### [API_SIGNATURES]\n## Public API Surface\n\n### Functions\n\n| Function | Parameters | Return Type |\n|----------|-----------|-------------|\n| `ApplyDefaults` | `cfg *Config` | (none) |\n| `ResolveConfig` | `repoRoot string`, `modelIDFlag string`, `numCtxFlag string` | `(*Config, error)` |\n\n### Types\n\n| Type | Definition Location |\n|------|---------------------|\n| `Config` | Not defined in this file (referenced from `internal/config/config.go`) |\n\n### Constants\n\n| Constant | Definition Location |\n|----------|---------------------|\n| `OllamaDefaultModelID` | Not defined in this file (referenced from `internal/config/config.go`) |\n| `OllamaDefaultBaseURL` | Not defined in this file (referenced from `internal/config/config.go`) |\n| `OllamaDefaultNumCtx` | Not defined in this file (referenced from `internal/config/config.go`) |\n| `DefaultDocsDir` | Not defined in this file (referenced from `internal/config/config.go`) |\n| `DefaultExtractionSteps` | Not defined in this file (referenced from `internal/config/config.go`) |\n| `DefaultSystemPrompt` | Not defined in this file (referenced from `internal/config/config.go`) |\n| `DefaultModuleSynthesisPrompt` | Not defined in this file (referenced from `internal/config/config.go`) |\n| `DefaultArchitecturePrompt` | Not defined in this file (referenced from `internal/config/config.go`) |\n| `DefaultQuickstartPrompt` | Not defined in this file (referenced from `internal/config/config.go`) |\n| `DefaultFileFactConsolidationPrompt` | Not defined in this file (referenced from `internal/config/config.go`) |\n| `CodeReducerModelIDEnvKey` | Not defined in this file (referenced from `internal/config/config.go`) |\n| `OllamaBaseURLEnvKey` | Not defined in this file (referenced from `internal/config/config.go`) |\n| `OllamaNumCtxEnvKey` | Not defined in this file (referenced from `internal/config/config.go`) |\n\n### Unexported Functions (Internal Only)\n\n| Function | Parameters | Return Type |\n|----------|-----------|-------------|\n| `deduplicate` | `a []string` | `[]string` |\n| `loadBaseConfig` | `repoRoot string` | `(*Config, error)` |\n| `resolveModelID` | `cfgVal string`, `flagVal string` | `string` |\n| `resolveBaseURL` | `cfgVal string` | `string` |\n| `parsePositiveInt` | `val string`, `errMsg string` | `(int, error)` |\n| `resolveNumCtx` | `cfgVal int`, `flagVal string` | `(int, error)` |\n\n#### [BUSINESS_LOGIC]\n**Primary Domain Problem:**\nResolve a configuration for an AI code generation pipeline by merging multiple sources (CLI flags, environment variables, YAML config file, system defaults) into a single consistent `Config` struct.\n\n**High-Level Algorithm Steps:**\n\n1. **Load base config** from a YAML file at the repository root. If the file does not exist, use an empty config struct (no error). Wrap any other load errors.\n\n2. **Apply system defaults** to any empty or zero-valued fields:\n - `ModelID`, `OllamaBaseURL`, `OllamaNumCtx`, `DocsDir`, `ExtractionSteps`, and all prompt fields (`SystemPrompt`, `ModuleSynthesisPrompt`, `ArchitecturePrompt`, `QuickstartPrompt`, `FileFactConsolidationPrompt`) are populated with their respective default constants when empty/zero.\n - `DocsDir` is additionally trimmed and path-cleaned.\n\n3. **Resolve `OllamaNumCtx`** with priority: CLI flag \u003e environment variable \u003e config value. Validates the result is a positive integer.\n\n4. **Deduplicate** the `Ignore` slice, preserving first-occurrence order.\n\n5. **Resolve `ModelID`** with priority: CLI flag \u003e environment variable \u003e config value.\n\n6. **Resolve `OllamaBaseURL`** with priority: environment variable \u003e config value.\n\n7. **Return** the fully resolved `Config` struct.\n\n#### [STATE_AND_CONCURRENCY]\nNo mutable state\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n**External I/O Interactions:**\n\n- **Disk I/O:** `loadBaseConfig` calls `LoadConfig(repoRoot)` (defined in `internal/config/io.go`), which reads a configuration file from the filesystem. If the file does not exist (`os.ErrNotExist`), the function returns an empty `Config{}` instead of an error. Any other read error is wrapped and returned.\n\n- **Environment Variables:** Three environment variables are read via `os.Getenv`:\n - `CodeReducerModelIDEnvKey` (in `resolveModelID`)\n - `OllamaBaseURLEnvKey` (in `resolveBaseURL`)\n - `OllamaNumCtxEnvKey` (in `resolveNumCtx`)\n\n- **No network, database, or API calls** are made by this file.\n\n**Error Propagation:**\n\n- `ResolveConfig` returns errors from `loadBaseConfig` (which wraps non-`os.ErrNotExist` errors from `LoadConfig`) and from `resolveNumCtx` (which wraps errors from `parsePositiveInt`).\n- `ApplyDefaults` and `deduplicate` swallow all errors silently.\n- `resolveModelID` and `resolveBaseURL` return values without errors; callers handle the empty-string case implicitly.\n- `parsePositiveInt` returns a formatted error string via `fmt.Errorf`.\n- Errors are propagated via Go's `error` return convention; no panics or exceptions are used." }, "internal/engine/cache.go": { - "sha256": "3caca4ede2968bdc9e9c51c3cae28229856b1f08e4c2679c5957f5fdd146293d", - "facts": "#### [API_SIGNATURES]\n# Public API Surface — cache.go (engine package)\n\n## Types\n\n| Element | Description | Parameters | Return Type |\n|---|---|---|---|\n| `FileCacheEntry` | Struct for file-level cache entries | — | — |\n| `MetadataCache` | Struct for metadata cache state | — | — |\n\n## Functions\n\n| Element | Description | Parameters | Return Type |\n|---|---|---|---|\n| `loadMetadataCache` | Loads or creates the metadata cache from disk | `repoRoot string`, `docsDir string` | `(*MetadataCache, error)` |\n| `IsInitialized` | Checks if the metadata cache file exists | `repoRoot string`, `docsDir string` | `bool` |\n\n---\n\n**Note:** Private functions (`computeStepsHash`, `newEmptyCache`, `saveMetadataCache`, `computeSHA256`) and private fields/methods are intentionally excluded from this list per Go naming conventions.\n\n#### [BUSINESS_LOGIC]\n## Primary Domain Problem\n\nThis code implements a **metadata cache for tracking extracted documentation facts** from source files in a repository. The engine persists per-file extraction results (\"facts\") along with file integrity hashes, enabling change detection and incremental reprocessing.\n\n---\n\n## High-Level Algorithm Steps\n\n1. **Initialize an empty cache structure** — creates versioned metadata container with maps ready to grow (files, modules).\n\n2. **Load existing cache from disk** — reads the serialized metadata file; if missing, returns a fresh empty cache. Handles incompatible cache versions by returning a clean slate rather than failing.\n\n3. **Compute per-file SHA256 hash** — hashes raw file content at its virtual path to produce an integrity fingerprint for change detection.\n\n4. **Persist cache state back to disk** — serializes the current cache (versioned, indented) and writes it atomically to a known metadata path.\n\n5. **Track extraction step history** — marshals the list of extraction steps into JSON and hashes them; this hash is stored in the top-level cache as `steps_hash`, enabling detection when the processing pipeline itself changed between runs.\n\n---\n\n## Domain Rules\n\n- Cache versioning: only cache versions matching `currentCacheVersion` (1) are accepted; mismatches yield a clean cache.\n- Graceful degradation: missing files and read failures return an empty cache instead of errors.\n- Nil safety: maps initialized to nil on load are replaced with fresh empty maps before use.\n- File hash is computed from raw content at the repository root using virtual paths (not relative).\n\n#### [STATE_AND_CONCURRENCY]\n**Mutable State:**\n\n- `MetadataCache.Version` — mutated in `saveMetadataCache()` via `cache.Version = currentCacheVersion`. This is a pointer receiver parameter; the caller's reference to the same struct value could be affected if they hold the returned pointer from `loadMetadataCache()`.\n\n- `MetadataCache.Files` and `MetadataCache.Modules` — mutated in `loadMetadataCache()` when either map is nil (reassigned to an empty initialized map). This mutates the local variable; the caller receives a new pointer value via return, so this is not shared mutable state from another goroutine's perspective unless the caller also holds a reference.\n\n- `currentCacheVersion` — package-level constant (`const`). Immutable by design, never modified in this file.\n\n**Synchronization Mechanisms:**\n\nNone detected. No locks (sync.Mutex, sync.RWMutex), no atomic types (`sync/atomic`), no async/await patterns, and no channel-based coordination visible in this file. All functions are synchronous with respect to shared data access.\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n## External I/O Interactions\n\nAll I/O operations funnel through the `tools.ReadFileSafely` and `tools.WriteFileSafely` helpers, with no network or database calls visible in this file.\n\n**Disk reads:**\n- **`loadMetadataCache`**: Reads a metadata cache JSON file from `\u003cdocsDir\u003e/\u003cmetadataFileName\u003e` relative to `repoRoot`. Returns an empty cache if the file does not exist (`os.ErrNotExist`). Other read errors are wrapped as \"failed to read metadata cache\" and returned.\n- **`IsInitialized`**: Calls `tools.ReadFileSafely(repoRoot, metadataPath)` to check existence of the metadata file. Returns `true` only when no error occurs (file exists). No explicit error return; uses boolean pattern.\n- **`computeSHA256`**: Reads a target file at `virtualPath` relative to `repoRoot`. If read fails, propagates the error as the second return value. Otherwise computes SHA256 of the content and returns hex string + nil error.\n\n**Disk writes:**\n- **`saveMetadataCache`**: Marshals cache struct to indented JSON, then calls `tools.WriteFileSafely(repoRoot, metadataPath, data)`. Returns marshaling errors directly; write errors propagate through `WriteFileSafely`.\n\n**No external side effects detected beyond disk I/O.**\n\n## Error Propagation\n\n| Function | Failure Mode | Handling Strategy |\n|---|---|---|\n| `computeStepsHash` | JSON marshal failure | Swallowed; returns `\"\"` silently. No error returned to caller. |\n| `loadMetadataCache` | File read failure (other than `os.ErrNotExist`) | Wrapped with \"failed to read metadata cache: %w\" and returned as error. |\n| `loadMetadataCache` | JSON unmarshal failure | Wrapped with \"failed to unmarshal metadata cache: %w\" and returned as error. |\n| `loadMetadataCache` | Version mismatch (`cache.Version != currentCacheVersion`) | Silently recovered; returns empty cache + nil error (treated as fresh start). |\n| `saveMetadataCache` | JSON marshal failure | Returned directly to caller. |\n| `saveMetadataCache` | File write failure | Propagated through `WriteFileSafely` to caller. No recovery logic here. |\n| `computeSHA256` | File read failure | Propagated as the second return value (error type). Caller handles via standard error check. |\n| `IsInitialized` | Any file issue | Returns `false`; no explicit error returned since function signature is bool only. |\n\n**Patterns observed:**\n- **Explicit recovery**: Only in `loadMetadataCache` for version mismatch and missing cache. Other errors propagate up to caller.\n- **Silent swallowing**: `computeStepsHash` discards marshal errors entirely; no indication of failure downstream." + "sha256": "177ebf55a88cf8283fd05837aa2a5e63b20c21489e2484edb0c8f3d982ad2a17", + "facts": "#### [API_SIGNATURES]\n## Public API Surface\n\n### Types\n\n- `FileCacheEntry` struct\n - Fields: `SHA256` (string), `Facts` (string)\n\n- `MetadataCache` struct\n - Fields: `Version` (int), `StepsHash` (string), `Files` (map[string]FileCacheEntry), `Modules` (map[string]string)\n\n### Functions\n\n- `loadMetadataCache(repoRoot string, docsDir string) (*MetadataCache, error)`\n- `IsInitialized(repoRoot string, docsDir string) bool`\n- `saveMetadataCache(repoRoot string, docsDir string, cache *MetadataCache) error`\n- `computeSHA256(repoRoot string, virtualPath string) (string, error)`\n\n#### [BUSINESS_LOGIC]\n**Core Purpose:**\nThis file implements a persistent metadata cache that tracks file state and configuration across engine runs. It enables the engine to detect when inputs have changed (by comparing stored hashes) and skip redundant work when the environment is identical to a previous run.\n\n**High-Level Algorithm Steps:**\n\n1. **Initialize or load cache** — Read a `.metadata.json` file from the docs directory. If missing, create a fresh cache with the current version. If the file exists but the version is incompatible, reset to a clean cache.\n\n2. **Compute configuration hash** — Hash the model ID, system prompt, synthesis prompt, architecture prompt, quickstart prompt, file fact consolidation prompt, and all extraction step names/prompts. This produces a single `steps_hash` that represents the current configuration state.\n\n3. **Track file state** — For each file, store its SHA256 content hash and associated facts. This allows the engine to compare current file contents against previously cached values.\n\n4. **Persist cache** — Write the metadata cache (version, steps hash, file entries, module mappings) to disk as indented JSON.\n\n5. **Check initialization** — Provide a boolean check to determine whether the cache file exists at all.\n\n#### [STATE_AND_CONCURRENCY]\nNo mutable state\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n**External I/O:**\n- Disk reads: `loadMetadataCache` reads `.metadata.json` from `docsDir` via `tools.ReadFileSafely`; `computeSHA256` reads arbitrary `virtualPath` via `tools.ReadFileSafely`\n- Disk writes: `saveMetadataCache` writes `.metadata.json` to `docsDir` via `tools.WriteFileSafely`\n- Disk existence check: `IsInitialized` checks `.metadata.json` existence via `tools.FileExistsSafely`\n- No network, database, or API interactions in this file\n\n**Error Propagation:**\n- `loadMetadataCache` returns `(*MetadataCache, error)`. Handles `os.ErrNotExist` by returning an empty cache (graceful degradation). Other read errors are wrapped with `fmt.Errorf` and returned.\n- `saveMetadataCache` returns `error`. Marshaling failures and write failures are returned directly.\n- `computeSHA256` returns `(string, error)`. Read errors are returned directly.\n- `IsInitialized` returns `bool`. No error return; existence check errors are handled internally by `tools.FileExistsSafely`.\n- `computeStepsHash` performs no I/O; no error propagation.\n- `newEmptyCache` performs no I/O; no error propagation.\n\n**Side Effects:**\n- Only disk I/O side effects (read/write/existence check on `.metadata.json` and arbitrary file paths)" }, "internal/engine/chunking.go": { - "sha256": "5170da5122537172eb0f79f6b32523fc212ca7fc02cff826b2faea34504e12de", - "facts": "#### [API_SIGNATURES]\nNo public elements were found in the provided file.\n\n#### [BUSINESS_LOGIC]\n**Primary Domain Problem**\nThe code addresses the limitation of Large Language Model (LLM) context windows when processing large volumes of text. It provides a mechanism to iteratively condense, synthesize, or deduplicate extensive datasets (such as architecture descriptions or file facts) by recursively applying reduction operations until the information fits within the model's capacity.\n\n**High-Level Algorithm Steps**\n1. **Expansion**: Split individual items that exceed the maximum character limit into smaller, overlapping segments.\n2. **Batching**: Group the items into batches such that the combined length of items in a single batch does not exceed the maximum character limit.\n3. **Recursive Reduction**:\n * If the items fit into a single batch, perform the LLM reduction operation (e.g., synthesis or consolidation) directly.\n * If multiple batches exist, recursively apply the reduction process to each batch to generate intermediate results.\n4. **Consolidation**: Combine the results from all reduced batches into a single set of text.\n5. **Convergence Check**: Compare the size of the consolidated intermediate results against the size of the input. If the reduction failed to significantly decrease the size (defined as being $\\ge$ 95% of the original size), stop the recursion and return the concatenated text to prevent infinite loops.\n6. **Iteration**: If the reduction was successful (size significantly decreased), treat the consolidated results as the new input and repeat the process from Step 3.\n\n#### [STATE_AND_CONCURRENCY]\nNo mutable state\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n### External I/O\n* **LLM API**: The `llmCaller.CallLLM` method is invoked to perform text synthesis and consolidation via an external service.\n* **Logging/Event Reporting**: The `LogEventFunc` is called to report status updates and progress during the reduction process.\n\n### Error Propagation\n* **Context Cancellation**: Errors from `ctx.Err()` are checked at the beginning of `reduceItems` and during iteration, propagating upward if the context is canceled or reaches its deadline.\n* **LLM Errors**: Errors returned by `llmCaller.CallLLM` are wrapped with a configuration-defined error message (`cfg.errMsg`) and propagated through the call stack.\n* **Validation Errors**: The `chunkTextWithOverlap` function returns errors if `maxRunes` is non-positive or if `overlapRunes` is greater than or equal to `maxRunes`.\n* **Recursive Propagation**: Errors encountered within the recursive `reduceItems` calls (including errors from `expandOversizedItems` and `reduceFn`) are returned through the stack to the original caller." - }, - "internal/engine/chunking_test.go": { - "sha256": "514b3558a78e579ed8549c97f00484f57ad106befdd3ff98516bd6a833ac7c12", - "facts": "#### [API_SIGNATURES]\nNo public API elements found in the provided source.\n\n#### [BUSINESS_LOGIC]\n**Core Purpose**\nThe domain problem involves partitioning text and sequences of items into bounded segments, specifically handling content continuity via overlapping segments and decomposing elements that exceed size constraints.\n\n**High-level Algorithm Steps**\n* Partition text into segments of a maximum length with a specified overlap between consecutive segments.\n* Decompose items that exceed a size threshold into smaller components.\n* Group items into batches subject to a size limit.\n* Aggregate items within a batch using a provided reduction function.\n* Calculate a size limit based on an input value that ensures a positive minimum.\n\n#### [STATE_AND_CONCURRENCY]\nNo mutable state\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n### External I/O\nNo external side effects.\n\n### Error Propagation\n* **`chunkTextWithOverlap`**: Returns an error as a second return value if `maxRunes \u003c= 0` or if `overlapRunes \u003e= maxRunes`.\n* **`reduceItems`**: Returns an error as a second return value.\n* **`expandOversizedItems`**: Returns an error as a second return value." + "sha256": "f94f579dcfa0aa756c071d70449f481f7fe83a4678c0aa424a1813462a897fe3", + "facts": "#### [BUSINESS_LOGIC]\n**Primary Domain Problem**\n\nThis code solves the problem of reducing and chunking large text content (code files, documentation, extracted facts) into manageable pieces for LLM processing, while preventing infinite reduction loops and maintaining information integrity.\n\n**High-Level Algorithm Steps**\n\n1. **Input Preparation**: Strip outer markdown code fences from text, filter out \"none\" facts (empty, \"none\", \"n/a\")\n\n2. **Limit Calculation**: Calculate maximum processing characters based on context window size minus system prompt overhead, with a minimum floor\n\n3. **Item Expansion**: Split oversized individual items into smaller chunks with overlap, then batch remaining items based on character limits\n\n4. **LLM Reduction**: Call LLM to reduce each batch, with partial batch instruction for summarizing intermediate results\n\n5. **Convergence Check**: If total output reaches 95% of total input, stop reducing and concatenate results (prevents infinite map-reduce loops)\n\n6. **Recursive Reduction**: If output is below 95% threshold, recursively reduce the intermediate results with the same parameters\n\n7. **Specialized Processing**: Separate paths for architecture synthesis (per node path) and file fact consolidation (per file and step), each with their own system prompts and instructions\n\n**Domain Rules**\n\n- Default chunk overlap: 800 characters\n- Max chars multiplier: 2\n- Loop prevention threshold: 95% of input\n- Minimum file limit: 1600 characters (overlap × 2)\n- Overlap must be between 0 and maxRunes-1\n- MaxRunes must be greater than 0\n\n#### [STATE_AND_CONCURRENCY]\nNo mutable state\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n**External I/O:**\n- `c.CallLLM(ctx, cfg.sysPrompt, prompt)` in `reduceWithLLM` — external API call to an LLM. Errors are wrapped with `cfg.errMsg` and returned as `error`.\n- `chunkTextWithOverlap` — no external I/O.\n- All other functions (`stripOuterMarkdownFence`, `calculateFileLimit`, `reduceInChunks`, `isNoneFact`, `reduceFileFacts`, `expandOversizedItems`, `batchItems`, `countRunes`, `reduceItems`) — no external I/O.\n\n**Error Propagation:**\n- `chunkTextWithOverlap` returns `error` for invalid `maxRunes` (≤ 0) or `overlapRunes` (out of range).\n- `reduceWithLLM` returns `error` wrapped with `cfg.errMsg` when `c.CallLLM` fails.\n- `reduceInChunks` propagates errors from `reduceWithLLM` directly.\n- `reduceFileFacts` propagates errors from `reduceWithLLM`; additionally returns `\"\"` if the LLM response is a \"none\" fact.\n- `reduceItems` returns errors from `expandOversizedItems` or `reduceFn`.\n- No panics are used. All errors are returned as values." }, "internal/engine/client.go": { - "sha256": "eae96ebc417be9d8e74557c74763ef4bdd5e7449a082a8952c6eb39f60700309", - "facts": "#### [API_SIGNATURES]\n# Public API Surface — `internal/engine/client.go`\n\n## Types (Exported)\n\n### `Message`\n\n```go\ntype Message struct {\n Role string // json:\"role\"\n Content string // json:\"content\"\n}\n```\n\n---\n\n## Unexported Elements (Internal Use Only)\n\n| Element | Type | Notes |\n|---------|------|-------|\n| `llmCaller` | interface | Has methods: `CallLLM`, `NumCtx` |\n| `llmClient` | struct | Embeds `*http.Client`; has method `NumCtx()` |\n| `newLLMClient(modelID, baseURL string, numCtx int)` | function | Returns `*llmClient` |\n| `ollamaRequest` | struct | Fields: `Model`, `Messages`, `Stream`, `Format`, `Options` |\n| `ollamaOptions` | struct | Field: `NumCtx` |\n| `ollamaResponse` | struct | Field: `Message` (type `Message`) |\n| `prepareOllamaRequest(ctx context.Context, systemPrompt string, messages []Message, jsonFormat bool)` | method on `*llmClient` | Returns `(*http.Request, error)` |\n| `CallLLM(ctx context.Context, systemPrompt string, messages []Message, jsonFormat bool)` | method on `*llmClient` | Returns `(string, error)` |\n\n#### [BUSINESS_LOGIC]\n**Primary Domain Problem:** \nThis code implements a client for invoking a large language model (LLM) through a remote HTTP endpoint, specifically targeting Ollama's `api/chat` protocol. It abstracts LLM interaction behind an interface (`llmCaller`) so the engine layer can depend on it without tying to a specific transport.\n\n**High-Level Algorithm Steps:**\n\n1. **Construct request payload** — Build an LLM call by prepending any system prompt to the user-provided message list, then serialize into Ollama's expected JSON schema (model ID, messages, stream flag, optional format/options).\n\n2. **Execute synchronous HTTP POST** — Send the serialized body to `baseURL/api/chat` using a context-aware client with a fixed timeout; no retries are attempted.\n\n3. **Parse response** — On 200 OK, deserialize the JSON into an Ollama-style envelope and return only the model's reply content as the result string.\n\n4. **Report non-OK status** — For any other HTTP status code, read a bounded slice of the error body and surface it as a formatted \"ollama api error\" with the status and response text.\n\n#### [STATE_AND_CONCURRENCY]\n**No mutable state**.\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n## External I/O Analysis\n\n**Network (HTTP → Ollama API)** — This module is a direct HTTP client targeting an external LLM provider. The call path is `POST {baseURL}/api/chat` with a JSON body constructed from the model ID, system prompt, and user messages. The response is read back as JSON and parsed into an `ollamaResponse`.\n\n**Timeout behavior** — The `http.Client` is initialized with `defaultHTTPTimeout`, referenced but not defined in this file. If that constant lives elsewhere or is missing, it will fail to compile; if present, requests exceeding the threshold are aborted by Go's `http.Client` transport layer.\n\n## Error Propagation\n\n| Failure point | Behavior |\n|---|---|\n| Request construction (`prepareOllamaRequest`) | Marshaling errors are wrapped with `\"failed to marshal request: %w\"`. New-request errors propagate as-is. Returned as `(string, error)` — caller decides. |\n| HTTP transport failure (`CallLLM` → `c.httpClient.Do(req)`) | Transport-level error (network down, DNS fail, timeout, etc.) is returned directly. No retry logic exists; the call fails fast per the inline comment. |\n| Successful status + parse error | Response body is fully read into memory, then unmarshaled. If JSON doesn't match `ollamaResponse`, a wrapped error `\"failed to parse response: %w\"` is returned. |\n| Non-200 status code | The body is read up to `maxErrorBodyBytes` (referenced but not defined in this file) and the content is embedded in an error string like `\"ollama api error: status {code}, response: {text}\"`. The actual `io.ReadAll` error from that branch is discarded (`body, _ := io.ReadAll(...)`). |\n\n## Notable Observations\n\n- **Error swallowed**: In the non-200 branch, if reading the response body fails (e.g., connection reset mid-stream), only the status code is reported; the underlying read failure is lost.\n- **`maxErrorBodyBytes` undefined in this file** — If this constant does not live elsewhere and is expected to be `int`, compilation will fail. The current branch reads an unbounded stream if the constant is missing, which could leak memory on large error responses from a hostile server.\n- **No retry / no circuit breaker** — Failures are returned immediately. Transient network errors are not retried; callers must handle their own backoff or resilience logic upstream." - }, - "internal/engine/constants.go": { - "sha256": "85ec25e43f84ebf8a1ce483e8efd9df8eb8fcf20fa2e8bbf3e243022a0730ae7", - "facts": "#### [API_SIGNATURES]\n# Public API Surface of `internal/engine/constants.go`\n\n## Result: No public elements found.\n\n- All 9 named constants (`defaultHTTPTimeout`, `maxErrorBodyBytes`, `defaultChunkOverlap`, `minNumCtxFloor`, `contextWindowAllocRatio`, `maxCharsMultiplier`, `metadataFileName`, `agentsFileName`, `defaultDirPerm`) are **unexported** (lowercase first letter).\n- No exported functions, methods, or types are defined in this file.\n\n#### [BUSINESS_LOGIC]\n## Core Purpose\n\nThis file defines operational configuration constants for an AI inference/agent engine runtime. It establishes fixed parameters that govern timeout behavior, error handling limits, context window management, and filesystem conventions used during agent execution.\n\n## High-Level Algorithm Steps\n\n1. **Set HTTP communication timeouts** — `defaultHTTPTimeout` (10 minutes) controls the maximum duration for remote API calls before they are aborted.\n\n2. **Cap error response payloads** — `maxErrorBodyBytes` (1 KB) limits how much error body data can be captured from failed requests.\n\n3. **Configure chunk overlap** — `defaultChunkOverlap` (800 tokens) defines the overlap between consecutive text chunks during streaming or processing.\n\n4. **Define context window floor** — `minNumCtxFloor` (512) sets a minimum threshold for context length calculations.\n\n5. **Allocate context windows proportionally** — `contextWindowAllocRatio` (0.75) reserves 75% of available space for primary content, leaving the remainder for metadata and overhead.\n\n6. **Scale character limits** — `maxCharsMultiplier` (3x) multiplies a base character limit to determine maximum allowed output length.\n\n7. **Establish filesystem conventions** — `metadataFileName`, `agentsFileName`, and `defaultDirPerm` define how state is persisted on disk during agent runs.\n\n## Domain Rules\n\n- All timeouts are absolute durations; no retry or backoff logic is implied by this file alone.\n- Error bodies are truncated at a fixed byte ceiling regardless of content type.\n- Chunk processing uses a fixed overlap value with no dynamic adjustment mentioned here.\n- Context window allocation follows a static 75/25 split ratio.\n- The `LogEventFunc` type is declared as an event callback signature but no events or logging system are implemented in this file.\n\n#### [STATE_AND_CONCURRENCY]\nNo mutable state\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n**External I/O:** No external side effects in this file. This file contains only constant declarations and a single function-type alias (`LogEventFunc`). It does not perform any disk, network, database, or API operations itself.\n\n**Error Propagation:** No error propagation mechanisms present in this file. There are no error types defined, no exception handling, and no error return values." - }, - "internal/engine/markdown.go": { - "sha256": "81bea4827161eb8978cd6f288e2276c2616da7832e8687400b6872d8d9c3749c", - "facts": "#### [API_SIGNATURES]\n## Public API Surface of `internal/engine/markdown.go`\n\nThis file contains **no exported (public) symbols**. All identifiers (`stripOuterMarkdownFence`, `markdownFenceRe`) begin with lowercase letters, which makes them unexported in Go.\n\n---\n\n### Internal Elements (for reference only — not part of the public API)\n\n- `var markdownFenceRe regexp.Regexp` — compiled regex for matching fenced code blocks\n- `func stripOuterMarkdownFence(content string) string` — strips surrounding markdown/json fences from input strings\n\n#### [BUSINESS_LOGIC]\n## Primary Domain Problem\n\n**Stripping surrounding code fence markers from markdown or JSON content.** This function cleans input that wraps code blocks in triple-backtick fences (with optional `markdown` or `json` language tags), exposing the raw interior content without the wrapping syntax.\n\n## High-Level Algorithm Steps\n\n1. **Trim whitespace** — Remove leading/trailing spaces and newlines from the input string to normalize boundaries before fence detection.\n2. **Match fenced pattern** — Check whether the trimmed string begins with three or more backticks (optionally followed by `markdown`/`json`) and ends with a matching set of backticks, capturing everything between them.\n3. **Extract inner content** — If the fence pattern matches, return the captured interior text after trimming whitespace around it.\n4. **Fallback pass-through** — If no fenced structure is detected, return the original input unchanged.\n\n#### [STATE_AND_CONCURRENCY]\n**Global/Package-Level Variables:**\n- `markdownFenceRe` — initialized once at package scope; never reassigned or modified anywhere in this file. Effectively read-only after compilation.\n\n**Synchronization Mechanisms:** None present (no mutexes, channels, async/await, or atomic types).\n\n**Conclusion:** No mutable state\n\n#### [ERRORS_AND_SIDE_EFFECTS]\nExternal side effects: None. The function performs no I/O operations (no file, network, database, or API interactions). It relies solely on standard library packages (`regexp`, `strings`).\n\nError propagation: No error return values are used. If the regex fails to match the input, the function returns the trimmed input string unchanged rather than signaling an error or panicking." + "sha256": "f73b361a9397f467cf8a35eed91c24a237dbb1c7b0312b007a4cc6a4c2aea85d", + "facts": "#### [API_SIGNATURES]\n# Public API Surface\n\n## Types\n\n- **`Message`** struct\n - `Role` string\n - `Content` string\n\n- **`llmCaller`** interface\n - `CallLLM(ctx context.Context, systemPrompt, userPrompt string) (string, error)`\n - `NumCtx() int`\n\n## Methods\n\n- **`llmClient.NumCtx()`**\n - Returns: `int`\n\n- **`llmClient.CallLLM(ctx context.Context, systemPrompt, userPrompt string)`**\n - Returns: `(string, error)`\n\n---\n\n**Note:** `llmClient` is an unexported type (lowercase). Only its exported methods are part of the public API. All internal types (`ollamaRequest`, `ollamaOptions`, `ollamaResponse`) and functions (`newLLMClient`, `prepareOllamaRequest`) are unexported.\n\n#### [BUSINESS_LOGIC]\n**Primary Domain Problem:**\nProvides a client for invoking a Large Language Model (LLM) through an HTTP API. Specifically targets the Ollama `/api/chat` endpoint to send system and user prompts and retrieve text responses.\n\n**High-Level Algorithm Steps:**\n1. Serialize a request payload containing the model ID, system prompt, user prompt, and configuration options (context length, temperature) into JSON.\n2. Send the serialized payload via HTTP POST to the configured base URL with a 10-minute timeout.\n3. On a 200 OK response, parse the JSON response to extract the model's text output.\n4. If the response indicates truncation (`done_reason: length`), emit a warning via the status callback.\n5. On any non-200 status, read up to 1024 bytes of the error body and return it as an error.\n6. Fail fast without retries on any HTTP error.\n\n#### [STATE_AND_CONCURRENCY]\nNo mutable state\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n**External I/O Interactions:**\n\n- **HTTP POST to Ollama API** (`/api/chat`): The `CallLLM` method serializes a JSON payload and sends it via `http.NewRequestWithContext` with a 10-minute timeout. The request body is a `bytes.NewReader` wrapping the marshaled `ollamaRequest` struct.\n\n- **Response reading**: On 200 OK, the full response body is read via `io.ReadAll`. On non-200 status, the body is read up to `maxErrorBodyBytes` (1024 bytes) via `io.LimitReader`.\n\n- **No retries**: The method is documented as \"failing fast without retries.\"\n\n- **Context cancellation**: The HTTP request is created with `ctx`, so cancellation propagates to the underlying connection.\n\n**Error Propagation:**\n\n- Marshaling errors from `prepareOllamaRequest` are wrapped with `fmt.Errorf(\"failed to marshal request: %w\", err)`.\n- Request creation errors and HTTP transport errors are returned directly.\n- Response parsing errors are wrapped with `fmt.Errorf(\"failed to parse response: %w\", err)`.\n- Non-200 responses return `fmt.Errorf(\"ollama api error: status %d, response: %s\", resp.StatusCode, string(body))`.\n- The `resp.Body.Close()` is deferred, so it executes regardless of error path.\n- The `onStatus` callback is invoked when `done_reason == \"length\"`, but this is a side effect, not an error path." }, "internal/engine/orchestrator.go": { - "sha256": "25b084a5d219794d3fe74159248a9584d50b319ead6ec6f7f152340f536e3dbb", - "facts": "#### [API_SIGNATURES]\n* `EventType` (type)\n* `EventStatus` (constant)\n* `EventError` (constant)\n* `Event` (type)\n * `Type` (EventType)\n * `Message` (string)\n\n#### [BUSINESS_LOGIC]\n**Primary Domain Problem**\nThe code orchestrates an automated pipeline for generating and maintaining hierarchical documentation for code repositories using Large Language Models (LLMs). It employs a Map-Reduce approach to transform source code into structured documentation, including architecture overviews, quickstart guides, and module-level summaries. The system manages incremental updates by tracking file changes through metadata caching and SHA256 hashing to avoid full regenerations when only specific subdirectories are modified.\n\n**High-Level Algorithm Steps**\n\n**Initialization (RunInit)**\n1. **Setup:** Load ignore patterns and the existing metadata cache.\n2. **Discovery:** Identify code files and compute their SHA256 hashes.\n3. **Tree Construction:** Build a hierarchical directory tree from the discovered files.\n4. **Hierarchical Synthesis:** Execute a Map-Reduce process to synthesize summaries across the tree.\n5. **Global Documentation:** Use an LLM to generate architecture and quickstart documentation based on the synthesized root summary.\n6. **Guidance Update:** Append AI agent guidelines to the repository to facilitate agent-based codebase navigation.\n7. **Teardown:** Persist the updated metadata cache.\n\n**Incremental Update (RunUpdate)**\n1. **Setup \u0026 Validation:** Initialize environment and verify if the metadata cache is invalidated by changes in the extraction configuration.\n2. **Change Detection:** Identify added, modified, or deleted files by comparing current hashes against the cache.\n3. **Cache Pruning:** Remove entries from the cache and the filesystem for directories no longer present in the current code tree.\n4. **Impact Analysis:** Determine which directories are affected by the detected changes and propagate these changes through the tree.\n5. **Selective Synthesis:** Perform hierarchical synthesis specifically for the affected directory branches.\n6. **Conditional Global Update:** Regenerate the root-level architecture and quickstart files only if the root directory is affected or if the files are missing.\n7. **Teardown:** Persist the updated metadata cache.\n\n#### [STATE_AND_CONCURRENCY]\n**Mutable State**\n* `cache (*MetadataCache)`: `StepsHash`, `Files`, `Modules`\n\n**Synchronization Mechanisms**\n* None\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n### External I/O\n* **LLM API**: Communicates with an LLM via `client.CallLLM` to receive text content for architecture and quickstart documentation.\n* **Disk/File System**:\n * **Read**: Loads `.gitignore` files via `tools.LoadGitignore`, the metadata cache via `loadMetadataCache`, agent guidelines via `tools.ReadFileSafely`, and code files via `computeSHA256`. It also performs existence checks via `os.Stat`.\n * **Write**: Generates documentation files (`architecture.md`, `quickstart.md`, module files, and agent guidelines) via `tools.WriteFileSafely`, creates the `modules` directory via `os.MkdirAll`, and saves the metadata cache via `saveMetadataCache`.\n * **Delete**: Removes stale module files via `os.Remove`.\n\n### Error Propagation\n* **Returned (Wrapped)**:\n * Errors from `client.CallLLM`.\n * Errors from `tools.WriteFileSafely`.\n * Errors from `tools.LoadGitignore` (except where the error is `os.IsNotExist`).\n * Errors from `loadMetadataCache`.\n * Errors from `os.MkdirAll`.\n * Errors from `tools.DiscoverCodeFiles`.\n * Errors from `synthesizeNode`.\n * Errors from `tools.ReadFileSafely` (if the subsequent attempt to write the file fails).\n* **Swallowed**:\n * `os.IsNotExist` errors when loading `.gitignore`.\n * Errors from `computeSHA256` (logged via `logEvent` and execution continues).\n * Errors from `saveMetadataCache` (logged via `logEvent`).\n * Errors from `os.Remove`." - }, - "internal/engine/orchestrator_test.go": { - "sha256": "dd30631b0853b00de0afec5a317e1d9b84c80471496c791a572500480a7e6cad", - "facts": "#### [API_SIGNATURES]\n* `DirNode` (struct)\n * `Path`\n * `Children` (map[string]*`DirNode`)\n* `EventType` (type)\n* `Event` (type)\n\n#### [BUSINESS_LOGIC]\n**Primary Domain Problem**\nThe code implements an orchestration engine designed to manage and update agent-specific guidelines or context based on file system structures and file content. It facilitates tracking changes in a directory tree and preparing documentation/guidelines for use by an LLM agent.\n\n**High-level Algorithm Steps**\n1. **Tree Traversal:** Iterate through a hierarchical directory structure to identify and mark all nodes as affected.\n2. **Content Hashing:** Scan a directory for specific files and compute unique identifiers (hashes) for existing files while ignoring missing ones.\n3. **Guideline Management:** Update or create agent guideline files in a specified directory, ensuring the operation is idempotent.\n4. **Orchestration Initialization:** Initialize the engine's lifecycle using a provided configuration, a target repository path, and an execution context.\n\n#### [STATE_AND_CONCURRENCY]\nNo mutable state\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n**External I/O**\n* **Disk**:\n * `t.TempDir()` is used to create temporary directories.\n * `os.WriteFile` is used to create `file1.txt` and `main.go`.\n * `updateAgentGuidelines` performs file creation and updates.\n * `os.ReadFile` is used to read file content.\n\n**Error Propagation**\n* **Error Return Codes**:\n * Errors returned by `os.WriteFile`, `updateAgentGuidelines`, `os.ReadFile`, and `o.RunInit` are intercepted by tests and result in `t.Fatalf` calls." - }, - "internal/engine/runner.go": { - "sha256": "78f573864a5dfacabfba38c9570672beaf629579334d25024f9662db3a2082ba", - "facts": "#### [API_SIGNATURES]\n# Public API Surface: `runner.go` (Module: `internal/engine`)\n\n## Types\n\n### `Mode`\n- **Kind:** Type alias (`string`)\n\n---\n\n## Constants\n\n### `ModeInit`\n- **Kind:** Constant of type `Mode`\n- **Value:** `\"init\"`\n\n### `ModeUpdate`\n- **Kind:** Constant of type `Mode`\n- **Value:** `\"update\"`\n\n---\n\n## Structs\n\n### `Runner`\n| Field | Type | Visibility |\n|-------|------|------------|\n| `cfg` | `*config.Config` | Private |\n\n---\n\n## Functions\n\n### `NewRunner(cfg *config.Config) *Runner`\n- **Kind:** Constructor / Factory function\n- **Parameters:**\n - `cfg *config.Config` — configuration instance\n- **Returns:**\n - `*Runner`\n\n---\n\n## Methods on `Runner`\n\n### `Run(ctx context.Context, repoRoot string, mode Mode, onEvent func(Event)) error`\n- **Kind:** Operation method\n- **Parameters:**\n - `ctx context.Context` — cancellation/context scope\n - `repoRoot string` — root directory of the repository\n - `mode Mode` — pipeline operation mode (`ModeInit` or `ModeUpdate`)\n - `onEvent func(Event)` — callback invoked for each event\n- **Returns:**\n - `error`\n\n#### [BUSINESS_LOGIC]\n**Primary Domain Problem**: This code manages and orchestrates an automated documentation generation pipeline for software projects, supporting both initial documentation creation (init mode) and incremental updates to existing documentation (update mode). It serves as the main entry point that coordinates AI-powered document processing across a repository.\n\n**High-Level Algorithm Steps**:\n1. Ensure project lockfile is added to `.gitignore` for version control safety\n2. Acquire an exclusive repository lock to prevent concurrent documentation operations \n3. Initialize LLM client and orchestrator based on provided configuration (model, base URL, context settings)\n4. Execute the appropriate pipeline mode: either run full initialization or incremental update using the configured AI model\n\n#### [STATE_AND_CONCURRENCY]\n## Mutable State Analysis for `runner.go`\n\n### Global Variables\n**None.** All state is scoped to struct instances or function parameters.\n\n### Shared States / Modified Properties\n- **`Runner.cfg *config.Config`**: Assigned once in `NewRunner()` and never modified after that. Not mutable post-initialization within this file.\n- No other struct fields are written to; no shared state is mutated by this code.\n\n### Synchronization Mechanisms\n- **Lock acquired via `security.AcquireLock(repoRoot)`** — a repository-level lock is acquired, then released via `defer lock.Unlock()`. This protects repository state during pipeline execution. The exact lock type (file-based, in-memory, etc.) and its thread-safety guarantees depend on the implementation in `internal/security`, which is outside this file's scope.\n\n### Summary\nThis code has **no mutable shared state** within its own boundary. It does introduce a synchronization primitive (`security.AcquireLock`) that wraps the execution of the documentation pipeline to guard against concurrent access to repository resources, but no internal fields are modified after initialization.\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n## External I/O Analysis\n\n### Filesystem Operations (Disk)\n\n| Operation | Location in Code | Purpose |\n|-----------|------------------|---------|\n| `tools.EnsureGitignoreHasLockfile(repoRoot)` | Line 53 | Writes/updates `.gitignore` to include the lockfile path. This is a **write** operation targeting disk. |\n| `security.AcquireLock(repoRoot)` → `lock.Unlock()` | Lines 60–65 | Acquires and releases a file-based lock (likely via `os.Flock` or similar). Involves both **read** (checking existing locks) and **write** (creating/removing the lockfile). |\n| Orchestrator pipeline (`orch.RunInit` / `orch.RunUpdate`) | Lines 71–84 | The orchestrator likely writes documentation files to disk during its execution. Specific I/O details are in other modules not provided here, but the runner delegates this entirely. |\n\n### Network Operations (API)\n\n| Operation | Location in Code | Purpose |\n|-----------|------------------|---------|\n| `newLLMClient(r.cfg.ModelID, r.cfg.OllamaBaseURL, r.cfg.OllamaNumCtx)` | Line 68 | Instantiates an LLM client configured with a model ID and Ollama base URL. The actual network I/O occurs downstream within the orchestrator's pipeline calls (`RunInit`/`RunUpdate`), not in this file directly. |\n\n### No External Side Effects\n\n- This file itself does **not** perform network calls, database operations, or API requests directly.\n- All external interactions are delegated to other modules: `config`, `security`, `tools`, and the orchestrator (which wraps an LLM client).\n\n---\n\n## Error Propagation Analysis\n\n### Returned Errors (Explicit)\n\n| Path | Behavior |\n|------|----------|\n| `EnsureGitignoreHasLockfile` fails → logs warning as event, **no return** | Swallowed; caller does not see the error. |\n| `AcquireLock` fails → wraps with `fmt.Errorf(\"failed to acquire repository lock: %w\", err)` and **returns immediately**. | Caller receives wrapped error. |\n| Orchestrator pipeline (`RunInit`/`RunUpdate`) returns non-nil → returned as-is to caller. | Propagated directly. |\n| Unsupported mode detected → wraps with `fmt.Errorf(\"unsupported mode: %s\", mode)`. | Caller receives descriptive error. |\n\n### Swallowed Errors (Silent)\n\n- **Gitignore warning** at line 53 is logged via the event callback but never returned or panicked. The runner continues execution regardless of this failure.\n- Any errors from `lock.Unlock()` are not checked (deferred), meaning lock cleanup errors are silently ignored." + "sha256": "7033691f944367911a656ed67a9ff72d55dae67b32e139339da3b32c74fd2771", + "facts": "#### [API_SIGNATURES]\n## Public API Surface\n\n### Types\n\n- **`RunMode`** (string type)\n - `ModeInit` = `\"init\"`\n - `ModeUpdate` = `\"update\"`\n\n- **`StatusFunc`** (function type)\n - Signature: `func(string)`\n\n- **`orchestrator`** (struct)\n - Fields:\n - `client` (type: `llmCaller`)\n\n- **`pipelineState`** (struct)\n - Fields:\n - `client` (type: `llmCaller`)\n - `repoRoot` (type: `string`)\n - `cfg` (type: `*config.Config`)\n - `cache` (type: `*MetadataCache`)\n - `affectedDirs` (type: `map[string]bool`)\n - `precalculatedHashes` (type: `map[string]string`)\n - `onStatus` (type: `StatusFunc`)\n\n### Functions\n\n- **`Run`**\n - Parameters: `ctx` (type: `context.Context`), `repoRoot` (type: `string`), `cfg` (type: `*config.Config`), `mode` (type: `RunMode`), `onStatus` (type: `StatusFunc`)\n - Returns: `error`\n\n- **`generateDoc`** (method on `orchestrator`)\n - Parameters: `ctx` (type: `context.Context`), `repoRoot` (type: `string`), `relPath` (type: `string`), `sysPrompt` (type: `string`), `promptHeader` (type: `string`), `contextItems` (type: `[]string`), `onStatus` (type: `StatusFunc`)\n - Returns: `error`\n\n- **`buildProjectContextItems`**\n - Parameters: `rootSum` (type: `string`), `tree` (type: `*DirNode`), `cache` (type: `*MetadataCache`)\n - Returns: `[]string`\n\n- **`GenerateStandardDocs`** (method on `orchestrator`)\n - Parameters: `ctx` (type: `context.Context`), `repoRoot` (type: `string`), `docsDir` (type: `string`), `rootSum` (type: `string`), `tree` (type: `*DirNode`), `cache` (type: `*MetadataCache`), `cfg` (type: `*config.Config`), `onStatus` (type: `StatusFunc`)\n - Returns: `error`\n\n- **`setupPipeline`**\n - Parameters: `repoRoot` (type: `string`), `cfg` (type: `*config.Config`)\n - Returns: `string`, `[]string`, `*MetadataCache`, `error`\n\n- **`teardownPipeline`**\n - Parameters: `repoRoot` (type: `string`), `docsDir` (type: `string`), `cache` (type: `*MetadataCache`), `onStatus` (type: `StatusFunc`), `successMsg` (type: `string`)\n - Returns: no value\n\n- **`hashCodeFiles`**\n - Parameters: `repoRoot` (type: `string`), `codeFiles` (type: `[]string`), `onStatus` (type: `StatusFunc`)\n - Returns: `[]string`, `map[string]string`\n\n- **`collectTreeDirs`**\n - Parameters: `tree` (type: `*DirNode`)\n - Returns: `map[string]bool`\n\n- **`updateAgentGuidelines`**\n - Parameters: `repoRoot` (type: `string`), `docsDir` (type: `string`)\n - Returns: `error`\n\n- **`RunInit`** (method on `orchestrator`)\n - Parameters: `ctx` (type: `context.Context`), `repoRoot` (type: `string`), `cfg` (type: `*config.Config`), `onStatus` (type: `StatusFunc`)\n - Returns: `error`\n\n- **`pruneStaleCache`**\n - Parameters: `repoRoot` (type: `string`), `docsDir` (type: `string`), `cache` (type: `*MetadataCache`), `tree` (type: `*DirNode`), `currentFiles` (type: `map[string]string`)\n - Returns: no value\n\n- **`detectFileChanges`**\n - Parameters: `repoRoot` (type: `string`), `ignores` (type: `[]string`), `cache` (type: `*MetadataCache`), `onStatus` (type: `StatusFunc`)\n - Returns: `[]FileChange`, `map[string]string`, `[]string`, `error`\n\n- **`RunUpdate`** (method on `orchestrator`)\n - Parameters: `ctx` (type: `context.Context`), `repoRoot` (type: `string`), `cfg` (type: `*config.Config`), `onStatus` (type: `StatusFunc`)\n - Returns: `error`\n\n### Constants\n\n- `agentsFileName` = `\"AGENTS.md\"`\n- `ModeInit` = `\"init\"`\n- `ModeUpdate` = `\"update\"`\n\n#### [BUSINESS_LOGIC]\n**Primary Domain Problem:**\nAutomatically generate and maintain codebase documentation (architecture overview, quickstart guide, module details) by analyzing the code structure and content through an LLM-based pipeline, with change detection to only regenerate affected parts on updates.\n\n**High-Level Algorithm Steps:**\n\n1. **Pipeline Initialization**: Load `.gitignore`, build ignore list, load metadata cache for state persistence\n2. **Code Discovery \u0026 Tree Building**: Discover all code files, compute SHA256 hashes, construct a directory tree structure\n3. **Cache Pruning**: Remove stale cache entries for files/directories that no longer exist in the current tree\n4. **Change Detection** (Update mode): Compare current file hashes against cached hashes to identify added, modified, or deleted files\n5. **Affected Module Propagation**: Determine which modules are affected by detected changes\n6. **Hierarchical Synthesis**: Use LLM to synthesize a root summary of the entire codebase\n7. **Standard Documentation Generation**: Generate `architecture.md` and `quickstart.md` using the synthesized root summary\n8. **Agent Guidelines Update**: Append/update `AGENTS.md` with references to generated documentation\n9. **Cache Persistence**: Save metadata cache for future runs\n\n#### [STATE_AND_CONCURRENCY]\n**Mutable State:**\n\n- `orchestrator.client` (type `llmCaller`) — set once in `Run()` and not modified thereafter.\n- `MetadataCache` (pointer) — passed by value as a pointer and mutated in place across function calls (e.g., `cache.Files = make(...)`, `cache.Modules = make(...)`, `cache.StepsHash = ...`).\n- `tree *DirNode` — passed by pointer and modified in place (e.g., `pruneStaleCache` deletes entries from `cache.Files` and `cache.Modules`).\n- `pipelineState` — created fresh per invocation of `RunInit`/`RunUpdate`; not shared across goroutines.\n\n**Synchronization Mechanisms:**\n\n- None. No mutexes, channels, atomic types, or goroutine-based concurrency are used. The `security.AcquireLock` is a file-level lock, not a goroutine synchronization primitive.\n\n**Thread Safety:**\n\n- No concurrent access patterns are visible. All state mutations occur sequentially within a single goroutine.\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n**External I/O Interactions:**\n\nAll I/O in this file is disk-based. No network, API, or database calls are present.\n\n- **Lock file**: `security.AcquireLock(repoRoot)` acquires a lock file; `defer lock.Unlock()` releases it. If acquisition fails, the defer still runs (unlocks a non-existent file).\n- **Gitignore**: `tools.LoadGitignore(repoRoot)` reads `.gitignore`; `tools.EnsureGitignoreHasLockfile(repoRoot)` writes to it.\n- **Code discovery**: `tools.DiscoverCodeFiles(repoRoot, ignores)` reads source files to enumerate them.\n- **Hashing**: `computeSHA256(repoRoot, f)` reads each code file to compute its SHA-256 hash.\n- **Documentation generation**: `tools.WriteFileSafely(repoRoot, relPath, []byte(doc))` writes generated docs (architecture.md, quickstart.md, module files).\n- **Agent guidelines**: `tools.ReadFileSafely(repoRoot, agentsFileName)` reads `AGENTS.md`; `tools.WriteFileSafely` writes it (either fresh or appended).\n- **Cache**: `loadMetadataCache` / `saveMetadataCache` read/write cache files in the docs directory.\n- **Stale cleanup**: `os.Remove(p)` removes stale module doc files; `os.Remove(subdirs[i])` removes empty directories.\n- **Path resolution**: `security.SafeResolve(repoRoot, filepath.Join(docsDir, \"modules\"))` resolves the modules directory path.\n\n**Error Propagation:**\n\n- **Fatal errors** (returned to caller): `security.AcquireLock` failure, `tools.LoadGitignore` failure, `tools.DiscoverCodeFiles` failure, `tools.WriteFileSafely` failure, `setupPipeline` errors.\n- **Swallowed as warnings** (logged via `onStatus`, not returned): `tools.EnsureGitignoreHasLockfile` failure, `computeSHA256` failure, `tools.ReadFileSafely` failure (triggers fallback write), `os.Remove` failure, `security.SafeResolve` failure.\n- **Silently ignored**: `os.Remove` errors in `pruneStaleCache` are discarded via `_ = os.Remove(...)`.\n- **Lock release on failure**: `defer lock.Unlock()` executes even if `AcquireLock` returns an error, attempting to unlock a file that may not exist." }, "internal/engine/synthesize.go": { - "sha256": "addf50cde81573976ea40816fabc5e2518d4e7db0787a017b53c9df4812963e9", - "facts": "#### [API_SIGNATURES]\nNo exported elements were found in the provided source code.\n\n#### [BUSINESS_LOGIC]\n**Primary Domain Problem**\nThe code implements a hierarchical documentation synthesis engine. It automates the generation of high-level technical summaries for a codebase by recursively traversing a directory tree and using a Large Language Model (LLM) to interpret file contents and consolidate them into structured, multi-level documentation (files $\\rightarrow$ subsystems $\\rightarrow$ modules).\n\n**High-Level Algorithm Steps**\n1. **Recursive Traversal (Bottom-Up):** Traverse the directory tree, processing child nodes before parent nodes to enable hierarchical summarization.\n2. **Cache \u0026 Change Detection:** Determine if a directory or file requires re-processing by checking if the path is marked as \"affected\" or if the file's SHA256 hash has changed.\n3. **File Fact Extraction:**\n * Split large files into overlapping chunks based on the LLM's context window.\n * Invoke the LLM for each configured extraction step (e.g., logic, types, etc.) across all chunks.\n * Consolidate chunked LLM responses into a single set of structured facts for the file.\n4. **Component Aggregation:** Collect the summarized facts from all files within a directory and the synthesized summaries from all immediate subdirectories.\n5. **Hierarchical Reduction:** Pass the aggregated list of file and subsystem summaries to the LLM to synthesize a single, cohesive summary for the current directory.\n6. **Persistence:** Write the synthesized summary to a Markdown file in the designated documentation directory and update the local cache.\n\n#### [STATE_AND_CONCURRENCY]\n**Mutable State**\n* `p.cache.Files` (map within `MetadataCache`)\n* `p.cache.Modules` (map within `MetadataCache`)\n\n**Synchronization Mechanisms**\n* None\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n### External I/O\n* **Disk**:\n * `tools.ReadFileSafely` is used to read file content from `p.repoRoot`.\n * `tools.WriteFileSafely` is used to write module documentation to a path constructed from `p.cfg.DocsDir` and `node.Path`.\n* **API**:\n * `p.client.CallLLM` is used to interact with an LLM.\n\n### Error Propagation\n* **Swallowed Errors**:\n * In `extractFileFacts`, errors returned by `tools.ReadFileSafely` are logged via `p.logEvent` and the function returns `nil` for the error.\n* **Propagated Errors**:\n * In `extractFileFacts`, errors from `chunkTextWithOverlap`, `p.client.CallLLM`, and `reduceFileFacts` are returned to the caller.\n * In `collectComponents`, errors from `extractFileFacts` are returned to the caller.\n * In `synthesizeNode`, errors from `p.ctx.Err()`, `synthesizeChildren`, `collectComponents`, `reduceInChunks`, and `tools.WriteFileSafely` are returned to the caller." - }, - "internal/engine/synthesize_test.go": { - "sha256": "d02ccd059de42bd048a4705e8f9517c422734aee699133be321c601e78835d1e", - "facts": "#### [API_SIGNATURES]\n* `TestSynthesizeNode(t *testing.T)`\n\n#### [BUSINESS_LOGIC]\n**Primary Domain Problem**\nGenerating hierarchical summaries of directory structures using Large Language Models (LLMs), utilizing caching and change detection to optimize performance during incremental updates.\n\n**High-level Algorithm Steps**\n1. Evaluate if a node is empty.\n2. Determine if the node has a valid cache entry and is not within the set of affected directories.\n3. Retrieve the cached summary if a valid, unaffected entry exists.\n4. Synthesize a new summary for nodes containing files and/or child nodes.\n\n#### [STATE_AND_CONCURRENCY]\nMutable state: `cache.Modules`\nSynchronization mechanisms: None\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n**External I/O**\n* **Disk**: \n * `t.TempDir()` creates a temporary directory.\n * `os.MkdirAll` creates a directory path (`modulesDir`) on the filesystem.\n * `os.WriteFile` writes a sample file (`main.go`) to the filesystem.\n* **Network/API**: No actual network or API calls occur; `mockLLMCaller.CallLLM` returns a hardcoded string and a `nil` error.\n\n**Error Propagation**\n* **Filesystem Errors**: Errors from `os.MkdirAll` and `os.WriteFile` result in a call to `t.Fatalf`, which terminates the test execution.\n* **Function Return Values**: `synthesizeNode` returns an `error` as its second return value, which is checked by the caller using `t.Fatalf` if non-nil." + "sha256": "d4f1f752800bbaf8de92f5555680c9a9e945ae915b5ba852e2fd712a298ebd58", + "facts": "#### [API_SIGNATURES]\n## Public API Surface\n\n### Types\n\n| Type | Description |\n|------|-------------|\n| `pipelineState` | Struct containing: `client llmCaller`, `repoRoot string`, `cfg *config.Config`, `cache *MetadataCache`, `affectedDirs map[string]bool`, `precalculatedHashes map[string]string`, `onStatus StatusFunc` |\n\n### Functions\n\n| Function | Signature |\n|----------|-----------|\n| `toSafeMarkdownFilename` | `func toSafeMarkdownFilename(modulePath string) string` |\n| `buildExtractionPrompt` | `func buildExtractionPrompt(filePath, siblingList, chunk string, step config.ExtractionStep) string` |\n| `maxExtractionOverhead` | `func maxExtractionOverhead(sysPrompt, filePath, siblingList string, steps []config.ExtractionStep) string` |\n| `extractFileFacts` | `func extractFileFacts(ctx context.Context, p *pipelineState, f string, siblingFiles []string) (string, error)` |\n| `synthesizeChildren` | `func synthesizeChildren(ctx context.Context, p *pipelineState, node *DirNode) ([]string, error)` |\n| `collectComponents` | `func collectComponents(ctx context.Context, p *pipelineState, node *DirNode, subsystems []string) ([]string, error)` |\n| `synthesizeNode` | `func synthesizeNode(ctx context.Context, p *pipelineState, node *DirNode) (string, error)` |\n\n### Constants\n\n| Constant | Description |\n|----------|-------------|\n| `defaultChunkOverlap` | Used in `extractFileFacts` for chunk overlap calculation (defined elsewhere) |\n\n### Notes\n\n- `FileCacheEntry` is referenced in `extractFileFacts` but defined in `internal/engine/cache.go`\n- `llmCaller`, `StatusFunc`, `MetadataCache` are referenced but defined in other files\n- `config.ExtractionStep`, `config.Config`, `config.SystemPrompt` are referenced but defined in `internal/config`\n\n#### [BUSINESS_LOGIC]\n**Domain Problem:**\nThe code implements a hierarchical codebase synthesis system that processes a directory tree structure to extract, consolidate, and cache documentation summaries for code modules.\n\n**High-Level Algorithm Steps:**\n\n1. **File Fact Extraction:** For each file, the system extracts structured facts using LLM calls with configurable extraction steps. Results are cached based on file SHA256 hash to avoid redundant processing.\n\n2. **Recursive Subsystem Synthesis:** Child directories are processed recursively first, synthesizing their contents into subsystem summaries before processing individual files.\n\n3. **Component Collection:** After subsystems are synthesized, individual file facts are collected. Both subsystem summaries and file facts are combined into a unified component list.\n\n4. **Hierarchical Consolidation:** All components (subsystems + files) for a directory are reduced and consolidated into a final summary using chunk-based processing.\n\n5. **Cache and Persistence:** Final summaries are cached in memory and written to markdown files in a `modules/` directory structure. If a directory is not affected and has a cached summary, it is reused without reprocessing.\n\n6. **Cleanup:** If a directory produces no components, its cached summary is deleted and the corresponding markdown file is removed.\n\n#### [STATE_AND_CONCURRENCY]\n**Mutable State:**\n\n- `pipelineState.cache.Files` (map[string]FileCacheEntry) — modified in `extractFileFacts` via `p.cache.Files[f] = FileCacheEntry{...}`.\n- `pipelineState.cache.Modules` (map[string]string) — modified in `synthesizeNode` via `p.cache.Modules[node.Path] = finalSum`.\n\n**Synchronization:**\n\n- No locks, mutexes, atomic types, or channel-based synchronization are used.\n- No global variables or shared state outside the `pipelineState` struct.\n- Functions accept `context.Context` for cancellation, but no concurrency primitives are present.\n\n**Assessment:**\n\nState is encapsulated in the `pipelineState` struct passed by reference. No external synchronization mechanisms are visible in this file.\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n## External I/O and Error Propagation Analysis\n\n### External I/O Interactions\n\n**Disk Read:**\n- `tools.ReadFileSafely(p.repoRoot, f)` — reads source files from the repository root. Called in `extractFileFacts`.\n\n**Disk Write/Delete:**\n- `tools.WriteFileSafely(p.repoRoot, modulePath, []byte(finalSum))` — writes synthesized module documentation to disk. Called in `synthesizeNode`.\n- `tools.RemoveFileSafely(p.repoRoot, modulePath)` — deletes stale module documentation files. Called in `synthesizeNode` when no components are found.\n- `saveMetadataCache(p.repoRoot, p.cfg.DocsDir, p.cache)` — persists metadata cache to disk. Called in `synthesizeNode`.\n\n**Network:**\n- `p.client.CallLLM(ctx, p.cfg.SystemPrompt, userPrompt)` — sends prompts to an LLM API. Called in `extractFileFacts` for each extraction step per chunk.\n\n### Error Propagation\n\n**Swallowed Errors:**\n- File read failures in `extractFileFacts`: `tools.ReadFileSafely` error is logged via `p.onStatus` and the function returns `(\"\", nil)`. The caller receives no error.\n\n**Propagated Errors:**\n- LLM call failures in `extractFileFacts`: returned as `(\"\", fmt.Errorf(\"LLM error extracting %s for %s: %w\", ...))`.\n- Chunking failures in `extractFileFacts`: returned as `(\"\", fmt.Errorf(\"failed to chunk file %s: %w\", ...))`.\n- Context cancellation in `synthesizeNode` and `collectComponents`: returned as `(\"\", err)`.\n- Write failures in `synthesizeNode`: returned as `(\"\", fmt.Errorf(\"failed to write module documentation for %s: %w\", ...))`.\n- Metadata cache save failures in `synthesizeNode`: returned as `(\"\", err)`.\n- `extractFileFacts` errors propagate through `collectComponents` and `synthesizeNode`.\n- `synthesizeChildren` errors propagate through `synthesizeNode`.\n\n**No external side effects** for any other operations in the provided code." }, "internal/engine/tree.go": { - "sha256": "8c1f570071d9444c7a768beb8d5afc19b827310cc889c30024cc1d6594b485cf", - "facts": "#### [API_SIGNATURES]\n* `ChangeStatus` (type, `string`)\n* `StatusAdded` (constant, `ChangeStatus`)\n* `StatusModified` (constant, `ChangeStatus`)\n* `StatusDeleted` (constant, `ChangeStatus`)\n* `FileChange` (type)\n * `Path` (string)\n * `Status` (ChangeStatus)\n* `DirNode` (type)\n * `Path` (string)\n * `Files` ([]string)\n * `Children` (map[string]*DirNode)\n\n#### [BUSINESS_LOGIC]\n**Primary Domain Problem**\nThe code identifies which directories in a file tree are \"affected\" by file system changes. This is used to determine which module-related documentation or metadata needs to be recalculated or updated based on file modifications, deletions, or missing documentation files.\n\n**High-level Algorithm Steps**\n1. **Tree Construction**: Convert a list of file paths into a hierarchical tree structure of directory nodes.\n2. **Change Mapping**: Identify specific changed file paths and mark the parent directories of any deleted files as affected.\n3. **Node Impact Detection**: Traverse the tree to identify nodes that are affected due to:\n * A missing entry in a metadata cache.\n * The presence of a changed file within the node.\n * The absence of a corresponding documentation file on the filesystem.\n4. **Upward Propagation**: Recursively propagate the \"affected\" status from child nodes up to their ancestor directories.\n\n#### [STATE_AND_CONCURRENCY]\nNo mutable state\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n**External I/O**\n* **Disk:** The `isNodeAffected` function performs a file system check using `os.Stat` to determine if a specific file path exists on disk.\n\n**Error Propagation**\n* **`security.SafeResolve`:** Errors returned by this function are caught; if an error is non-nil, the function skips the subsequent disk check and returns `false`.\n* **`os.Stat`:** \n * If `os.IsNotExist(err)` evaluates to `true`, the function returns `true`.\n * If `os.Stat` returns a non-nil error that is not a \"not exist\" error, the error is swallowed and the function returns `false`." - }, - "internal/engine/tree_test.go": { - "sha256": "8343fd8c2fc3747b2c5b0562ce72cc41e72e27e8ee1d3b43699c6370eb1b9270", - "facts": "#### [API_SIGNATURES]\n```markdown\n# Public API Surface (inferred from test references)\n\n## Functions\n\n### buildTree\n- **Parameters**: `files []string` (file paths to index)\n- **Returns**: `*DirNode` — root tree node containing `.Path`, `.Children map[string]*DirNode`, and `.Files []string` fields\n\n### determineAffected \n- **Parameters**: \n - `tree *DirNode`\n - `tempDir string`\n - `docs string` (repository/docs directory path)\n - `cache Cache` — interface with `.Modules map[string]string` field\n - `changes []FileChange` — slice of struct containing `.Path string` and `.Status StatusModified` fields\n- **Returns**: `map[string]bool` — affected module paths\n\n### propagateAffected\n- **Parameters**: \n - `tree *DirNode`\n - `affected map[string]bool`\n- **Returns**: `map[string]bool` — propagated affected status per path\n\n## Types (inferred from usage)\n\n| Type | Fields Observed |\n|------|------------------|\n| `*DirNode` | `.Path string`, `.Children map[string]*DirNode`, `.Files []string` |\n| `Cache` | `.Modules map[string]string` |\n| `FileChange` | `.Path string`, `.Status StatusModified` (enum-like) |\n```\n\n---\n\n⚠️ **Note**: This surface was extracted from a test file (`tree_test.go`). Actual definitions, signatures, and error handling reside in the corresponding source files (e.g., `tree.go`).\n\n#### [BUSINESS_LOGIC]\n## Primary Domain Problem\n\nThis code solves **module dependency impact analysis** for a project's source tree. It answers two questions: (1) \"Given these files, what is the hierarchical directory structure?\" and (2) \"When one file changes, which modules are affected—and does that affect their parent directories too?\"\n\n## High-Level Algorithm Steps\n\n### Step 1 — Build Tree\n- Take a flat list of project file paths.\n- Group them into nested `DirNode` structures: files at the same depth share a directory node; subdirectories become children of their parent.\n- Each `DirNode` stores its own path and the files contained directly within it.\n\n### Step 2 — Determine Affected Modules\n- Walk the tree while tracking file changes (e.g., modifications).\n- Mark any module whose source file changed as \"affected.\"\n- Unrelated branches remain unaffected.\n\n### Step 3 — Propagate Affected Status Upward\n- After marking individual modules, propagate the affected status to parent directories.\n- If a child is affected, its parent becomes affected too (transitive propagation up the tree).\n- Sibling branches that did not change stay clean—only ancestors of changed files are marked.\n\n#### [STATE_AND_CONCURRENCY]\nNo mutable state\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n## External I/O Interactions\n\n**TestBuildTree:**\n- **No external side effects.** This test only constructs a tree structure in memory and validates it with assertions. No file system, network, or database interactions occur.\n\n**TestDetermineAndPropagateAffected:**\n- **File system I/O via standard library (`os`, `path/filepath`).** The test creates a temporary directory using `t.TempDir()`, then writes dummy files to simulate module existence so that the production code's `os.Stat` calls succeed:\n - `os.MkdirAll(...)` — creates nested directories under the temp dir.\n - `os.WriteFile(...)` — writes `\"test\"` bytes into each module path.\n\n## Error Propagation\n\n- **No error propagation observed.** All test functions invoke production methods (`buildTree`, `determineAffected`, `propagateAffected`) without checking their return values or handling errors. Any errors they produce are swallowed (unhandled/ignored).\n- Assertions use `t.Errorf` and `assert.True` patterns; failures do not propagate to callers because these are leaf-level tests with no caller chain visible here." - }, - "internal/engine/utils.go": { - "sha256": "9b8cd943c9fdbb73451b01584d7d76b73d92194f133ec07c04517ac3ca7f713b", - "facts": "#### [API_SIGNATURES]\n## Public API Surface of `internal/engine/utils.go`\n\n### Functions\n\n| Function | Parameters | Return Type |\n|---|---|---|\n| **`toSafeMarkdownFilename`** | `modulePath string` | `string` |\n| **`makeLogEvent`** | `onEvent func(Event)` | `LogEventFunc` |\n\n\u003e *Note: Types `LogEventFunc`, `EventType`, and `Event` are referenced but not defined within this file. Their definitions must be resolved from the importing package.*\n\n#### [BUSINESS_LOGIC]\n## Core Purpose\n\nThis file provides lightweight utility helpers for the engine module: generating safe markdown filenames from directory paths, and creating adapter-style log event callbacks that support optional listeners.\n\n## High-Level Steps\n\n1. **Safe filename generation**: Given a module path string, return `\"README.md\"` if the path is empty or `.`; otherwise join the path with `\"README.md\"`.\n2. **Log event factory**: Return a function that, given an `EventType` and message string, forwards to a supplied callback only if it's non-nil (nil-safe adapter pattern).\n\n## Domain Rules\n\n- Empty/`.` module paths map to `\"README.md\"` by default.\n- Unknown or invalid callbacks are silently swallowed — no panic on nil listeners.\n\n#### [STATE_AND_CONCURRENCY]\nNo mutable state\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n**External I/O:** No external side effects. The only filesystem-related function (`toSafeMarkdownFilename`) constructs path strings via `filepath.Join` but performs no actual file operations (no open/read/write). Pure string manipulation.\n\n**Error Propagation:** None observed. Both functions return values without error types, and neither panics or returns errors." - }, - "internal/security/errors.go": { - "sha256": "ccabad7427e45855c7277be338dfd613a25148de1d42d9c284ba887432a5bacd", - "facts": "#### [API_SIGNATURES]\n## Public API Surface — `internal/security/errors.go`\n\n| Element | Type | Parameters | Returns |\n|---------|------|------------|---------|\n| `ErrPathTraversal` | `error` (variable) | — | — |\n| `ErrLockHeld` | `error` (variable) | — | — |\n\n#### [BUSINESS_LOGIC]\n## Domain Problem\n\nThis code provides error definitions for security operations that handle:\n- **Path traversal prevention** — detecting when file paths resolve outside an allowed root directory\n- **Cross-process lock management** — reporting when a shared resource is already locked by another process\n\n## High-Level Algorithm Steps\n\n1. A path-traversal check validates whether a given file or directory path resolves within the repository root boundary; if it does not, `ErrPathTraversal` is returned.\n\n2. A lock acquisition attempt detects contention from another process holding the same lock; if detected, `ErrLockHeld` is returned.\n\n#### [STATE_AND_CONCURRENCY]\nNo mutable state\n\n#### [ERRORS_AND_SIDE_EFFECTS]\nNo external side effects. This file contains only sentinel error declarations; no functions, imports beyond `errors`, or runtime operations exist here to perform disk, network, database, or API interactions.\n\nError propagation: None in this file. Both errors are pre-allocated sentinels awaiting use by other code in the module. Propagation of these errors (e.g., wrapping via `fmt.Errorf` or returning from a function) would occur elsewhere, not here." + "sha256": "4b8de575797b40f7b676ff8d4180a065733f0d2ba0482ff48c970b2795348cde", + "facts": "#### [API_SIGNATURES]\n## Public API Surface\n\n### Types\n\n- `ChangeStatus` — string type\n- `StatusAdded` — `ChangeStatus` constant\n- `StatusModified` — `ChangeStatus` constant\n- `StatusDeleted` — `ChangeStatus` constant\n- `FileChange` — struct with fields `Path string`, `Status ChangeStatus`\n- `DirNode` — struct with fields `Path string`, `Files []string`, `Children map[string]*DirNode`\n\n### Functions\n\n- `propagateAffected` — `func(node *DirNode, affectedDirs map[string]bool) map[string]bool`\n- `isNodeAffected` — `func(n *DirNode, changedFiles map[string]bool, repoRoot string, docsDir string, cache *MetadataCache) bool`\n- `determineAffected` — `func(node *DirNode, repoRoot string, docsDir string, cache *MetadataCache, filteredChanges []FileChange) map[string]bool`\n- `buildTree` — `func(files []string) *DirNode`\n\n#### [BUSINESS_LOGIC]\n**Core Purpose:** Build a hierarchical directory tree from file paths and determine which directories are affected by file changes.\n\n**Domain Rules:**\n\n1. Files are organized into a tree structure where each node represents a directory containing files and child directories.\n2. File changes are tracked with a status (Added, Modified, Deleted) and a path.\n3. For deleted files, all ancestor directories are marked as affected by walking up the path to the root.\n4. A node is considered affected if any of its files are in the changed set, or if its module does not exist in the cache.\n5. Affected status propagates upward through the tree: if any descendant is affected, the parent is also marked affected.\n6. The final result is a map of directory paths that are affected by the given file changes.\n\n**Algorithm Steps:**\n\n1. Parse each file path into its directory components and build a tree of `DirNode` objects.\n2. For each `FileChange`, mark the file path as changed; if status is Deleted, walk up the directory hierarchy marking each ancestor as affected.\n3. Traverse the tree recursively, checking each node against the changed files and cache to determine if it is affected.\n4. Propagate affected status from children to parents, collecting all affected directory paths.\n\n#### [STATE_AND_CONCURRENCY]\nNo mutable state.\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n**External I/O:**\n- `tools.FileExistsSafely` is called in `isNodeAffected` to check if a module file exists on disk. This is a disk I/O operation. The `tools` package is external to this file.\n\n**Error Propagation:**\n- No error return values in any function.\n- No panic calls.\n- No exception handling.\n- The behavior of `tools.FileExistsSafely` (whether it returns an error, panics, or swallows errors) is not visible in this file." }, "internal/security/security.go": { - "sha256": "c0d705dd9e8931f4ec613dd5d7fb8e54dfb3eb794891138f4715c4b15175235a", - "facts": "#### [API_SIGNATURES]\n# Public API Surface — `internal/security` package\n\n## Types\n\n### `SafeResolve`\n- **Signature**: `func SafeResolve(repoRoot, inputPath string) (string, error)`\n- **Description**: Cleans the input path and ensures it lies strictly inside the repository. Resolves symlinks on existing ancestor parts to prevent path traversal via symlinks.\n\n---\n\n### `SimpleLock` struct\n- **Fields** (unexported):\n - `lockPath string`\n - `file *os.File`\n - `mu sync.Mutex`\n - `closed bool`\n\n## Functions\n\n### `AcquireLock`\n- **Signature**: `func AcquireLock(repoRoot string) (*SimpleLock, error)`\n- **Description**: Acquires a simple file lock in the repoRoot. Uses O_EXCL to ensure atomicity.\n\n---\n\n### `Unlock` (method on `*SimpleLock`)\n- **Signature**: `func (l *SimpleLock) Unlock() error`\n- **Description**: Releases the lock by closing the file and removing it. Idempotent and thread-safe.\n\n---\n\n## Unresolved References (not defined within this file)\n- `ErrPathTraversal` — referenced in `SafeResolve` signature path validation but not declared here.\n- `ErrLockHeld` — referenced in `AcquireLock` error wrapping but not declared here.\n\n#### [BUSINESS_LOGIC]\n## Primary Domain Problem\n\nThis code solves two distinct problems within a repository context: preventing path traversal attacks and managing process-level locking to avoid stale locks blocking legitimate operations.\n\n## High-Level Algorithm Steps\n\n**SafeResolve (Path Traversal Prevention):**\n1. Compute the absolute root directory, resolving any symlinks on it first\n2. Join the resolved root with an input path candidate\n3. Walk up from that joined path until a physically existing ancestor is found — recording each skipped component along the way\n4. Resolve symlinks on that existing ancestor\n5. Reconstruct the full target path from the resolved ancestor plus all previously-skipped components\n6. Verify the reconstructed path remains inside the resolved root; reject if it escapes\n\n**SimpleLock (File-Based Process Locking):**\n1. Use `O_EXCL` flag to atomically create a lock file — failure means another process holds the lock or a stale lock exists, which is treated as an error condition requiring manual cleanup\n2. Write the current PID into the lockfile for identification purposes\n3. Release via idempotent unlock: close the file descriptor and remove the lockfile; always clean up even on error paths\n\n#### [STATE_AND_CONCURRENCY]\n### Mutable State Identified\n\n**Class-Level Properties (Struct Fields of `SimpleLock`)**\n\n| Field | Type | Mutability | Notes |\n|-------|------|------------|-------|\n| `lockPath` | `string` | **Modified once** in `AcquireLock`, then read-only thereafter. Not mutated by any other method. |\n| `file` | `*os.File` | **Modified once** in `AcquireLock`. Closed during `Unlock()`. No concurrent mutation observed. |\n| `mu` | `sync.Mutex` | Read via lock/unlock primitives only. Field itself is never assigned after struct initialization. |\n| `closed` | `bool` | Modified by `Unlock()` (set to `true`). Also read inside `Unlock()` for the idempotency check (`if l.closed { return nil }`). Concurrent calls serialize through `mu`. |\n\n**Synchronization Mechanisms**\n\n- **`sync.Mutex` (`l.mu`)**: Used in `SimpleLock.Unlock()`. Acquired at entry and released via `defer`. Protects the close-and-remove sequence, making `Unlock()` atomic with respect to itself.\n- **No package-level lock/mutex**: The struct owns its own mutex; there is no shared global lock across instances of `SimpleLock`.\n\n### Thread Safety Assessment\n\n- `Unlock()` is self-synchronizing via `mu.Lock()/defer mu.Unlock()`, making it safe for concurrent calls on the same instance.\n- Concurrent calls to `AcquireLock` on different instances each create their own independent `SimpleLock` with separate mutexes—no contention between instances is modeled or guaranteed by this code.\n- The file-level lock (the `.code-reducer.lock` file itself) provides OS-level mutual exclusion, which complements the in-memory `mu`.\n\n### No Package-Level Global Mutable State\n\nNo package-level variables are declared. All mutable state is encapsulated within individual `SimpleLock` instances.\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n## External I/O Analysis\n\nAll interactions are with the **local filesystem** (disk). No network, database, or API calls exist.\n\n### `SafeResolve` — Path Validation\n| Operation | External System | Purpose |\n|---|---|---|\n| `filepath.Abs(repoRoot)` | Disk (filesystem) | Resolve absolute path from relative input |\n| `filepath.EvalSymlinks(absRoot)` | Disk (filesystem) | Resolve symlinks on the root directory before joining |\n| `os.Lstat(current)` loop | Disk (filesystem) | Walk up ancestor chain, stopping at first physically existing parent |\n| `filepath.EvalSymlinks(current)` | Disk (filesystem) | Resolve symlinks on the found ancestor path |\n\n### `AcquireLock` — Lock Acquisition\n| Operation | External System | Purpose |\n|---|---|---|\n| Calls `SafeResolve(repoRoot, LockFileName)` | Disk | Resolve canonical lock file path inside repo root |\n| `os.OpenFile(lockPath, O_WRONLY\\|O_CREATE\\|O_EXCL, 0644)` | Disk | Atomic create-with-exclusion; returns error if another writer holds the file (OS-level atomicity) |\n| `f.Write([]byte(fmt.Sprintf(\"%d\\n\", os.Getpid())))` | Disk | Writes current process PID to lockfile for debugging/inspection |\n\n### `SimpleLock.Unlock()` — Lock Release\n| Operation | External System | Purpose |\n|---|---|---|\n| `l.file.Close()` | Disk (OS file descriptor) | Close the open file handle |\n| `os.Remove(l.lockPath)` | Disk | Remove the lockfile from disk |\n\n---\n\n## Error Propagation Analysis\n\n### SafeResolve\n- **Wraps** with `%w` on `filepath.Abs` and `EvalSymlinks(absRoot)` errors.\n- **Wraps** with `%w` on each `Lstat` failure that is not a \"not exist\" error.\n- **Wraps** with `%w` on the final `EvalSymlinks(current)`.\n- On path-traversal detection, returns a wrapped error using a custom sentinel (`ErrPathTraversal`) with `inputPath` as context — this **swallows** the original traversal signal and replaces it with a formatted message.\n\n### AcquireLock\n- **Propagates directly** (no wrapping) when `SafeResolve` fails.\n- On `OpenFile` failure: if error is `os.IsExist`, wraps with `%w` using custom sentinel (`ErrLockHeld`) plus context string describing stale lockfile; otherwise wraps the raw error with path info.\n- On **Write** failure (after successful OpenFile): closes file and removes it, then returns a wrapped error that **swallows** any close/remove errors into a single formatted message — only the original write error is surfaced, even if cleanup also failed.\n\n### SimpleLock.Unlock()\n- Returns `nil` if already closed (`l.closed == true`).\n- If not yet closed: closes file first; then attempts to remove lockfile. **Swallows** close errors when reporting removal errors (only surfaces the remove error if it fails)." - }, - "internal/security/security_test.go": { - "sha256": "c60ccd832bd3368a58529148b43c15672d32aa5aa6563beff3cf61ec3eb32a16", - "facts": "#### [API_SIGNATURES]\n# Public API Surface: internal/security\n\n## Functions\n\n### SafeResolve\n- **Parameters:**\n - `repoRoot` (string) – repository root path\n - `inputPath` (string) – input file/directory path to resolve\n- **Return values:**\n - resolved path (unverified type, inferred from usage pattern as string)\n - error\n\n### AcquireLock\n- **Parameters:**\n - `repoRoot` (string) – repository root path\n- **Return values:**\n - lock object (type: `Lock`, inferred from method calls in tests)\n - error\n\n## Types\n\n### Lock\n- **Method:**\n - `Unlock()` (no parameters, returns unverified type and error based on test usage pattern)\n\n#### [BUSINESS_LOGIC]\n## Primary Domain Problem\n\nThe code solves two concurrent concerns in a repository-aware context:\n\n1. **Secure path resolution** — resolving user-supplied paths against a known root while preventing directory traversal and symlink escape attacks.\n2. **Exclusive access control** — ensuring only one goroutine/process can operate on the protected resource at a time via file-based locking.\n\n## High-Level Algorithm Steps\n\n### SafeResolve\n1. Receive a candidate path string and an anchor (repository root) directory.\n2. Resolve the candidate to an absolute filesystem path, using the repository root as the base for relative inputs.\n3. Compare the resolved absolute path against the canonicalized repository root. If they differ — meaning the resolved path escapes outside the repository boundary — reject with an error.\n4. Return success when the resolved path remains within (or equals) the repository root.\n\n### AcquireLock / Unlock\n1. Create a lock file inside the protected resource directory and attempt to open it exclusively (`O_CREATE | O_EXCL | O_RDWR`).\n2. If the exclusive open fails, reject — another holder is already active.\n3. On success, return an unlock handle that releases the file when closed (via `syscall.Flock` with `LOCK_UN`).\n\n#### [STATE_AND_CONCURRENCY]\n**Mutable State:**\n- `AcquireLock(repoRoot)` — acquires a lock that tracks whether it is held (mutable internal state). Confirmed by subsequent call failing and requiring explicit `Unlock()`.\n- `lock1.Unlock()` / `lock2.Unlock()` — releases the acquired lock, modifying its internal held-state.\n\n**Thread Safety:**\n- Lock acquisition (`AcquireLock`) + unlock pattern suggests mutex-like synchronization is implemented (or simulated) by the security package.\n- No locks/mutexes are used within the test file itself; each `Test*` function runs independently in a single goroutine.\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n## External I/O and Error Propagation Analysis\n\n### External Systems Interacted With\n\nAll interactions in this file target the **local filesystem** via the standard library's `os` package:\n\n| Operation | Function | Purpose |\n|-----------|----------|---------|\n| Directory creation | `os.MkdirAll()` | Sets up test fixtures (`src/sub/`) |\n| File writing | `os.WriteFile()` | Creates `file.txt`, `..config` for traversal tests |\n| Symlink creation | `os.Symlink()` | Creates a symlink pointing outside the repo root (conditional) |\n\nThese are all **test-side** operations. The actual external I/O performed by the *production* code (`security.SafeResolve()`, `security.AcquireLock()`) is inferred from test behavior, not directly shown in this file.\n\n### Error Propagation Patterns\n\n**Test fixture setup errors**:\n- `os.MkdirAll()` / `os.WriteFile()` errors are propagated **up to the test framework** via `t.Fatal(err)`. This terminates the test immediately—no recovery or retry logic exists at the test level.\n\n**Symlink creation error**:\n- `os.Symlink()` error is swallowed and replaced with `t.Skip(\"Symlinks not supported on this OS/filesystem\")`. The test continues only if the operation fails, but no further I/O is attempted.\n\n**Production function errors (inferred from test assertions)**:\n- `security.SafeResolve(repoRoot, inputPath)` returns an error to its caller; tests verify both success and failure paths via `wantErr` flag comparison (`t.Errorf(...)`).\n- `security.AcquireLock(repoRoot)` returns `(lock, err)`. Test asserts that a second concurrent call fails with `t.Fatal(\"AcquireLock() should have failed when lock is already held\")` — indicating the production code signals contention via error return.\n\n**No panic/crash observed**. All errors follow Go conventions: errors are returned to callers; tests assert expected outcomes rather than relying on panics." + "sha256": "326856047b9569cab4dc9a2e5bf42ea238b5a4d501674bcf8085dfe4651bf4b0", + "facts": "#### [API_SIGNATURES]\n## Public API Surface\n\n### Types\n\n- **`SimpleLock`**\n - Fields: `lockPath string`, `file *os.File`, `mu sync.Mutex`, `closed bool`\n\n### Functions\n\n- **`SafeResolve(repoRoot, inputPath string) (string, error)`**\n - Cleans input path and ensures it lies strictly inside the repository root.\n\n- **`AcquireLock(repoRoot string) (*SimpleLock, error)`**\n - Acquires a simple file lock in the repoRoot using O_EXCL for atomicity.\n\n### Methods\n\n- **`(*SimpleLock) Unlock() error`**\n - Releases the lock by closing the file and removing it. Idempotent and thread-safe.\n\n### Constants\n\n- **`LockFileName`** = `\".code-reducer.lock\"`\n- **`defaultFilePerm`** = `0644`\n\n### Error Variables\n\n- **`ErrPathTraversal`** = `errors.New(\"security violation: path traversal detected\")`\n- **`ErrLockHeld`** = `errors.New(\"lock is already held by another process\")`\n\n#### [BUSINESS_LOGIC]\n**Primary Domain Problem**\n\nThis code provides two security mechanisms for a repository-level tool:\n\n1. **Path traversal prevention** — ensures any resolved path stays strictly inside the repository root, even when symlinks are involved.\n2. **Process-level locking** — prevents concurrent execution by multiple processes of the same tool.\n\n**High-Level Algorithm Steps**\n\n**SafeResolve:**\n1. Convert the repository root to an absolute path.\n2. Resolve symlinks on the repository root.\n3. Walk up the input path from the root, stopping at the first physically existing ancestor.\n4. Resolve symlinks on that existing ancestor.\n5. Rebuild the full path by appending the non-existent suffix components.\n6. Verify the rebuilt path is inside the resolved root. If not, return `ErrPathTraversal`.\n\n**AcquireLock:**\n1. Resolve the lock file path using `SafeResolve`.\n2. Open the file with `O_EXCL` (exclusive creation). If the file already exists, return `ErrLockHeld`.\n3. Write the current process PID to the file.\n4. Return a `SimpleLock` handle.\n\n**Unlock:**\n1. Acquire the internal mutex.\n2. If already closed, return nil.\n3. Close the file descriptor.\n4. Remove the lock file. If both close and remove fail, return the error from the non-nil one.\n\n#### [STATE_AND_CONCURRENCY]\n**Mutable State:**\n\n- `SimpleLock` struct fields: `closed` (bool) is modified in `Unlock()` (set to `true`). `mu` (`sync.Mutex`) is locked/unlocked in `Unlock()`.\n- No global mutable state. All constants (`LockFileName`, `defaultFilePerm`, `ErrPathTraversal`, `ErrLockHeld`) are immutable.\n\n**Synchronization:**\n\n- `SimpleLock.Unlock()` uses `sync.Mutex` (`l.mu`) to protect the `closed` field.\n- `AcquireLock()` is a package-level function that creates a new `SimpleLock` instance; it does not use any synchronization mechanism.\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n**External I/O Interactions:**\n\n- **Filesystem (disk):** All operations target the local filesystem.\n - `filepath.Abs` — resolves absolute path for `repoRoot`.\n - `filepath.EvalSymlinks` — resolves symlinks on `repoRoot` and on intermediate ancestor paths.\n - `os.Lstat` — checks existence of path components during traversal resolution.\n - `os.OpenFile` — opens the lock file with `O_WRONLY|O_CREATE|O_EXCL` for atomic lock acquisition.\n - `os.Remove` — removes the lock file during `Unlock()` and on write failure in `AcquireLock()`.\n - `os.Getpid` — reads process ID (no external I/O).\n - `filepath.Rel` — computes relative path to verify containment.\n\n- **No network, database, or API interactions.**\n\n**Error Propagation:**\n\n- `SafeResolve` returns `(string, error)`. Errors are wrapped with `fmt.Errorf` using `%w` for chainability. Sentinel errors `ErrPathTraversal` and `ErrLockHeld` are returned directly.\n- `AcquireLock` returns `(*SimpleLock, error)`. If `OpenFile` fails with `os.IsExist`, it wraps `ErrLockHeld`. If `Write` fails, the file is closed and removed before returning a wrapped error.\n- `Unlock` returns `error`. It is guarded by `l.mu` (mutex). If `l.closed` is true, returns nil. Otherwise closes the file, then attempts `os.Remove`. If close succeeds but remove fails (and is not `os.IsNotExist`), the remove error is assigned to `err`. If close fails, the remove error is ignored.\n- Errors are not logged or printed; they propagate to the caller. No panic/recover is used." }, "internal/tools/file_tools.go": { - "sha256": "77f21deb5529b9a10f13c99937ea63c0828a5db0e5295e6a9b8c8b1fa190d4e1", - "facts": "#### [API_SIGNATURES]\n# Public API Surface — `internal/tools/file_tools.go`\n\n## Functions\n\n### ReadFileSafely\n```go\nfunc ReadFileSafely(repoRoot string, virtualPath string) ([]byte, error)\n```\nReads a file by resolving its virtual path relative to the repository root. Returns the file content as bytes and any error encountered.\n\n---\n\n### WriteFileAtomic\n```go\nfunc WriteFileAtomic(targetPath string, data []byte, perm os.FileMode) error\n```\nWrites binary data atomically to `targetPath` using a temp-file-and-rename pattern. The directory is created with mode `0755`. Returns an error on failure; returns `nil` only after successful rename.\n\n---\n\n### WriteFileSafely\n```go\nfunc WriteFileSafely(repoRoot string, virtualPath string, content []byte) error\n```\nWrites file content by resolving the virtual path and delegating to `WriteFileAtomic`. Returns an error if path resolution or atomic write fails.\n\n---\n\n### EnsureGitignoreHasLockfile\n```go\nfunc EnsureGitignoreHasLockfile(repoRoot string) error\n```\nEnsures that `.gitignore` contains a line matching `security.LockFileName`. Reads the existing file, appends the lock entry if absent (adding a newline before it when needed), and writes back atomically. Returns an error on read/write failures.\n\n---\n\n### LoadGitignore\n```go\nfunc LoadGitignore(repoRoot string) ([]string, error)\n```\nReads `.gitignore` from `repoRoot`, strips comments (`#`) and blank lines, and returns the remaining patterns as a `[]string`. Returns `nil, nil` when no `.gitignore` exists; returns an error on other read failures.\n\n---\n\n### ShouldIgnoreFile\n```go\nfunc ShouldIgnoreFile(relPath string, gitIgnore *ignore.GitIgnore) bool\n```\nReturns `true` if the given relative path should be ignored: matches via the provided GitIgnore object (if non-nil), contains a dot-prefixed component, or ends with `.egg-info`.\n\n---\n\n### DiscoverCodeFiles\n```go\nfunc DiscoverCodeFiles(repoRoot string, ignores []string) ([]string, error)\n```\nRecursively walks `repoRoot` to collect relative file paths that pass ignore filtering. Skips directories whose names start with `.`, end with `.egg-info`, or match the compiled GitIgnore rules. Returns a list of valid relative paths and any walk error.\n\n#### [BUSINESS_LOGIC]\n## Primary Domain Problem\n\nThe code provides safe file I/O utilities for analyzing a code repository's structure. It solves two problems: (1) reliably reading/writing files within a known repo root with TOCTOU protection, and (2) discovering which files in the tree are \"code\" versus build artifacts, dependencies, or lockfiles.\n\n## High-Level Algorithm Steps\n\n**Safe file operations:**\n- Resolve any virtual path against a fixed `repoRoot` before touching the filesystem\n- Use temp-file + rename pattern for writes to prevent partial/corrupt state if interrupted\n- Sync and close before finalizing to guarantee data reaches storage\n\n**Lockfile handling:**\n- Ensure `.code-reducer.lock` is recorded in `.gitignore` so it never appears as a source file during discovery\n\n**Gitignore loading:**\n- Parse `.gitignore`, strip comments and blank lines, return the active pattern list\n\n**Ignore detection (two layers):**\n1. Apply user-supplied ignore patterns plus compiled `.gitignore` rules to any given path\n2. Independently reject paths whose components start with `.` or end with `.egg-info`\n\n**Code discovery:**\n- Walk the entire directory tree from `repoRoot`\n- Skip directories matching ignore rules, dot-prefixed names (e.g. `.build`, `.venv`), or `.egg-info` suffixes\n- For files, run both ignore-checks and collect those that pass\n\n#### [STATE_AND_CONCURRENCY]\nNo mutable state\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n## External I/O \u0026 Error Propagation Analysis: `file_tools.go`\n\n---\n\n### 1. Disk Read/Write Operations (POSIX file system)\n\nAll functions in this module interact exclusively with the local filesystem via standard OS calls (`os.ReadFile`, `os.CreateTemp`, `os.MkdirAll`, `os.Rename`, `os.Chmod`, `os.Open`). No network, database, or external API interactions exist.\n\n| Function | I/O Type | External System |\n|---|---|---|\n| `ReadFileSafely` | Disk read (`os.ReadFile`) | Local filesystem — target path resolved through `security.SafeResolve` before reading |\n| `WriteFileAtomic` | Disk write (create temp → sync → chmod → rename) | Local filesystem — writes to a `.tmp.*` intermediate file then renames into place |\n| `WriteFileSafely` | Disk read + disk write | Delegates path resolution and atomic write; no additional I/O beyond what the two sub-functions perform |\n| `EnsureGitignoreHasLockfile` | Disk read (`os.ReadFile`) + disk write (`WriteFileAtomic`) | Reads `.gitignore`, appends lockfile pattern, writes back atomically |\n| `LoadGitignore` | Disk read (`os.Open` + `bufio.Scanner`) | Reads `.gitignore`; returns `nil, nil` when file does not exist (no error) |\n| `DiscoverCodeFiles` | Disk walk (`filepath.WalkDir`) | Recursively walks the repository root; skips directories matching ignore rules or dot-prefixed names |\n\n---\n\n### 2. Error Propagation Patterns\n\nErrors are propagated as Go typed errors in all paths except one notable case:\n\n- **`ReadFileSafely`** — wraps `os.ReadFile` error with context (\"failed to read file content\"). The path resolution error from `security.SafeResolve` is passed through unwrapped.\n- **`WriteFileAtomic`** — each step (mkdir, create temp, write, sync, close, chmod, rename) returns a wrapped error. The deferred cleanup block runs on any failure path but does not propagate; errors are returned directly to the caller.\n- **`WriteFileSafely`** — unwrapped `security.SafeResolve` error propagates straight through. Otherwise defers to `WriteFileAtomic`'s return value.\n- **`EnsureGitignoreHasLockfile`** — `.gitignore` read errors (except `os.IsNotExist`) are wrapped with \"error reading .gitignore\". A non-existent `.gitignore` falls through to the write path, which will then fail if the directory does not exist; no special handling for that intermediate state.\n- **`LoadGitignore`** — returns `nil, nil` when file is absent (treated as success). Any other error from `os.Open` propagates unwrapped. Scanner errors are appended via `scanner.Err()` at the end — if a scan error occurs mid-stream but `lines` has already been populated, it still gets returned to the caller.\n- **`ShouldIgnoreFile`** — no I/O; returns only boolean values. No error path.\n- **`DiscoverCodeFiles`** — walk errors are wrapped with context (path + \"error walking\"). Relative-path computation failures are also wrapped. Directory entries that match ignore rules return `filepath.SkipDir`. Non-matching files append to the slice. If the walk fails at any point, the returned error is passed through as-is after wrapping.\n\n---\n\n### 3. Notable Observations\n\n- **TOCTOU mitigation**: `WriteFileAtomic` uses a temp file + rename pattern, which avoids race conditions on the target path during writes.\n- **Sync before close**: `tmpFile.Sync()` is called before `Close()`, ensuring data reaches stable storage before the handle is released — relevant for crash safety but adds latency per write.\n- **Deferred cleanup in WriteFileAtomic**: On any error, `tmpFile.Close()` and `os.Remove(tmpName)` run via defer. This prevents partial temp file leakage on failure paths. The flag `success` gates whether to trigger cleanup.\n- **No panic/recover anywhere** — all errors are returned as typed values. No `panic`, no bare recovery blocks observed in this module." - }, - "internal/tools/file_tools_test.go": { - "sha256": "dc63ebefde774e1f683301b191a1fcbbc7d904dc209eb84af5a9c510bea05889", - "facts": "#### [API_SIGNATURES]\n# Public API Surface — `internal/tools` package\n\n## Functions\n\n| Function | Parameters | Returns |\n|---|---|---|\n| **WriteFileSafely** | `(repoRoot string, relPath string, content []byte)` | error |\n| **LoadGitignore** | `(repoRoot string)` | (patterns `[]string`, error) |\n| **ShouldIgnoreFile** | `(relPath string, gitIgnore interface{})` | bool |\n| **DiscoverCodeFiles** | `(repoRoot string, ignores []string)` | ([]string, error) |\n\n---\n\n\u003e **Notes:** \n\u003e - Function signatures are inferred strictly from the test file's usage patterns. Return types for multi-value returns are deduced from how results are consumed in tests (e.g., `err != nil` checks, slice length comparisons). \n\u003e - The exact signature of `ShouldIgnoreFile`'s second parameter (`gitIgnore`) is confirmed as a compiled gitignore object per the test's import and usage.\n\n#### [BUSINESS_LOGIC]\n**Primary Domain Problem:** \nThis code solves the problem of reliably discovering source-code files within a repository while correctly applying `.gitignore` rules — filtering out build artifacts, hidden files/directories, and anything explicitly ignored by gitignore patterns. It also provides safe file-writing utilities that integrate with this discovery workflow.\n\n**High-Level Algorithm Steps:**\n\n1. **WriteFileSafely** – Atomically writes a target file (likely using temp-file + rename to avoid partial reads).\n2. **LoadGitignore** – Scans for `.gitignore` in the repo root, parses its contents into ignore patterns (skipping blank lines and comments).\n3. **ShouldIgnoreFile** – Given a relative path and an `ignore.IgnoreList`, returns whether the file matches any ignore pattern — used to filter out build outputs, hidden paths, etc.\n4. **DiscoverCodeFiles** – Walks the directory tree, collects files that are Go source code (`.go` extension), excludes anything matched by `.gitignore` patterns or hidden-file rules, and returns the resulting list of discovered source files.\n\n#### [STATE_AND_CONCURRENCY]\nNo mutable state\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n## External I/O and Error Propagation Analysis\n\n**All interactions: Local filesystem (disk) only.** No network, API, database, or other external system interactions observed.\n\n---\n\n### Filesystem Operations by Test Function\n\n#### `TestWriteFileSafely`\n| Operation | Source Code | Purpose |\n|-----------|-------------|---------|\n| Write to disk | `tools.WriteFileSafely(repoRoot, \"src/new_file.txt\", []byte(\"hello\"))` | Creates a file under repo root |\n| Read from disk | `os.ReadFile(filepath.Join(repoRoot, \"src\", \"new_file.txt\"))` | Verifies written content |\n\n#### `TestLoadGitignore`\n| Operation | Source Code | Purpose |\n|-----------|-------------|---------|\n| Read .gitignore (missing) | `tools.LoadGitignore(repoRoot)` — first call | Tests graceful handling of missing file |\n| Write to disk | `os.WriteFile(filepath.Join(repoRoot, \".gitignore\"), content, 0644)` | Creates test `.gitignore` fixture |\n| Read .gitignore (exists) | `tools.LoadGitignore(repoRoot)` — second call | Verifies parsed patterns |\n\n#### `TestShouldIgnoreFile`\n**No external I/O.** Uses pre-compiled in-memory gitignore (`ignore.CompileIgnoreLines`).\n\n#### `TestDiscoverCodeFiles`\n| Operation | Source Code | Purpose |\n|-----------|-------------|---------|\n| Create directories | `os.MkdirAll(filepath.Join(repoRoot, d), 0755)` × 3 | Sets up test directory tree (`src`, `src/.hidden`, `build`) |\n| Write files | `os.WriteFile(..., []byte(\"test\"), 0644)` × 5 | Populates fixture files in various locations |\n| Read filesystem | `tools.DiscoverCodeFiles(repoRoot, ignores)` | Discovers code files while filtering ignored paths |\n\n---\n\n### Error Propagation Patterns\n\nAll test functions propagate errors directly to the testing framework via **fatal-level assertions** — no recovery or fallback logic:\n\n| Test Function | Error Path | Handler |\n|---------------|-----------|---------|\n| `TestWriteFileSafely` | Write error → `t.Fatalf(\"WriteFileSafely() failed: %v\", err)` | Aborts test, reports failure with message |\n| `TestWriteFileSafely` | Read error → `t.Fatalf(\"Failed to read created file: %v\", err)` | Aborts test after write succeeded |\n| `TestLoadGitignore` | Load error (missing file) → `t.Fatalf(\"LoadGitignore() failed when file is missing: %v\", err)` | Aborts test |\n| `TestLoadGitignore` | Write fixture error → `t.Fatal(err)` | Aborts test silently |\n| `TestLoadGitignore` | Load error (exists) → `t.Fatalf(\"LoadGitignore() failed: %v\", err)` | Aborts test |\n| `TestDiscoverCodeFiles` | MkdirAll errors → `t.Fatal(err)` × 3 | Aborts test silently |\n| `TestDiscoverCodeFiles` | WriteFile errors → `t.Fatal(err)` × 5 | Aborts test silently |\n| `TestDiscoverCodeFiles` | Discover error → `t.Fatalf(\"DiscoverCodeFiles() failed: %v\", err)` | Aborts test with message |\n\n**Pattern**: Errors are never recovered, deferred, or logged. All failures terminate the test immediately via `t.Fatal`/`t.Fatalf`. No panic recovery observed in this file." + "sha256": "76e3b2d96a906dd00f1d19e5e09adabe467495057732a0fdf2ac5070ff7ba84e", + "facts": "#### [API_SIGNATURES]\n## Public API Surface\n\n### Functions\n\n| Function | Parameters | Return Type |\n|---|---|---|\n| `FileExistsSafely` | `repoRoot string`, `virtualPath string` | `bool` |\n| `RemoveFileSafely` | `repoRoot string`, `virtualPath string` | `error` |\n| `ReadFileSafely` | `repoRoot string`, `virtualPath string` | `[]byte`, `error` |\n| `WriteFileAtomic` | `targetPath string`, `data []byte`, `perm os.FileMode` | `error` |\n| `WriteFileSafely` | `repoRoot string`, `virtualPath string`, `content []byte` | `error` |\n| `EnsureGitignoreHasLockfile` | `repoRoot string` | `error` |\n| `LoadGitignore` | `repoRoot string` | `[]string`, `error` |\n| `ShouldIgnoreFile` | `relPath string`, `gitIgnore *ignore.GitIgnore` | `bool` |\n\n### Types\n\nNone\n\n### Constants\n\nNone\n\n### Variables\n\nNone\n\n#### [BUSINESS_LOGIC]\n**Primary Domain Problem:**\nSafely discover, read, write, and manage source code files within a repository while preventing path traversal attacks and excluding non-code files (build artifacts, dependencies, tests, binaries, and documentation).\n\n**High-Level Algorithm Steps:**\n\n1. **Path Traversal Prevention** — All file operations resolve virtual paths through a security layer before accessing the filesystem.\n\n2. **Atomic File Writes** — New or updated files are written to a temporary file first, then renamed to the target path to avoid partial writes or race conditions.\n\n3. **Gitignore Integration** — `.gitignore` patterns are loaded and used to filter out ignored paths during file discovery.\n\n4. **Recursive File Discovery** — Walks the repository directory tree, collecting relative paths of regular files that pass all filters.\n\n5. **Multi-Layer File Filtering** — Each discovered file is checked against:\n - Predefined skip lists (specific filenames, extensions, test suffixes)\n - Gitignore patterns\n - Dot-prefixed path components (hidden files/directories)\n - Binary file detection (null byte check in first 512 bytes)\n\n6. **Lockfile Management** — Ensures the `.code-reducer.lock` file is included in `.gitignore` to prevent it from being committed.\n\n#### [STATE_AND_CONCURRENCY]\nGlobal variables:\n- `skippedFileNames` (map[string]bool) — initialized once, never modified\n- `skippedExtensions` (map[string]bool) — initialized once, never modified\n- `skippedTestSuffixes` ([]string) — initialized once, never modified\n\nSynchronization mechanisms: None\n\nMutable state: None\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n**External I/O Interactions:**\n- **Disk I/O only.** All operations use `os.*` functions: `Stat`, `Remove`, `ReadFile`, `CreateTemp`, `MkdirAll`, `Rename`, `Chmod`, `Open`, `Read`.\n- `filepath.WalkDir` recursively traverses the filesystem tree.\n- `ignore.CompileIgnoreLines` parses ignore patterns (in-memory, no external I/O).\n- `security.SafeResolve` resolves virtual paths (internal package, no external I/O).\n- No network, database, or API calls.\n\n**Error Propagation:**\n- **Swallowed (returns false):** `FileExistsSafely` (swallows `os.Stat` error), `isBinaryFile` (swallows `os.Open` error).\n- **Propagated as error:** `RemoveFileSafely`, `ReadFileSafely`, `WriteFileAtomic`, `WriteFileSafely`, `EnsureGitignoreHasLockfile`, `LoadGitignore`, `DiscoverCodeFiles`.\n- **Propagated as error with check:** `RemoveFileSafely` ignores `os.IsNotExist` but returns other errors.\n- **Propagated as error with wrap:** `WriteFileAtomic` wraps all sub-errors with `fmt.Errorf(\"failed to %s: %w\", ...)`.\n- **Propagated as error with skip:** `DiscoverCodeFiles` wraps `filepath.WalkDir` errors with path context.\n- **No panics.** All errors return as values." }, "internal/tools/git_tools.go": { - "sha256": "bdf47dd0db3277f300f4775e1dfb68f549dd24b9f78a2a884984bb6c930f8264", - "facts": "#### [API_SIGNATURES]\n## Public API Surface: `tools` package (`git_tools.go`)\n\n- **`RunGit(repoRoot string, args ...string) (string, error)`** — Executes a git command in the specified repo directory with `--no-pager`. Returns trimmed stdout and any error.\n\n- **`VerifyGitRepo(repoRoot string) error`** — Checks that `git` is available and that `repoRoot` is inside a git working tree.\n\n#### [BUSINESS_LOGIC]\n**Primary Domain Problem**\nThe code provides an abstraction for executing Git commands within a specific directory and validating whether that directory constitutes a recognized Git repository. It centralizes process execution logic to ensure consistent handling of stdout, stderr, and error reporting when interacting with the Git version control system.\n\n**High-Level Algorithm Steps**\n\n1. **Initialize Command:** Construct a Git command invocation in the specified root directory, appending `--no-pager` to any user-provided arguments to suppress pager output.\n2. **Execute and Capture:** Run the constructed process while capturing standard output and error streams into buffers.\n3. **Handle Success or Failure:**\n * If execution succeeds, trim whitespace from the captured standard output and return it as a string result.\n * If execution fails, trim whitespace from the captured error stream and return a formatted error message containing the failure details.\n4. **Validate Repository Status:** Invoke `git rev-parse --is-inside-work-tree` specifically to check if the directory is inside a work tree. If this command returns an error, classify the location as not being a Git repository and return that specific classification.\n\n#### [STATE_AND_CONCURRENCY]\nNo mutable state\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n### External Systems Interacted With\n\n| System | Mechanism | Details |\n|--------|-----------|---------|\n| **External process (git)** | `exec.Command(\"git\", ...)` via `os/exec` | Spawns a child `git` subprocess in `repoRoot`. Stdout captured to `bytes.Buffer`, stderr separately buffered. `cmd.Run()` executes the command and returns the exit error. |\n\n### Error Propagation\n\n- **Primary entry point — `RunGit`**:\n - Returns `(string, error)`. If `cmd.Run()` fails, it wraps the error with: \n `fmt.Errorf(\"git command failed: %v, stderr: %s\", err, trimmedErr)` \n The original `err` from `exec.Cmd` (e.g., `*exec.Error`, non-zero exit status) is embedded as `%v`; stderr text is appended. No panic occurs on failure.\n\n- **Secondary entry point — `VerifyGitRepo`**:\n - Calls `RunGit(repoRoot, \"rev-parse\", \"--is-inside-work-tree\")`. If the returned error is non-nil, it wraps with: \n `fmt.Errorf(\"not a git repository (or any of the parent directories): %w\", err)` \n Uses `%w` for `wrapError`-compatible wrapping (recoverable by unwrappers).\n\n### No External Side Effects\n\nNo direct disk reads/writes, no network calls, no database interactions. The only external interaction is subprocess execution against the installed `git` binary." + "sha256": "075026a1fde408cef2e049152151e73e7cbfba8da0fcc970d600faab69a166c4", + "facts": "#### [API_SIGNATURES]\n- `VerifyGitRepo(repoRoot string) error`\n\n#### [BUSINESS_LOGIC]\n**Primary Domain Problem:**\nVerify whether a given directory path resides inside a git repository.\n\n**High-Level Algorithm Steps:**\n1. Execute `git rev-parse --is-inside-work-tree` in the specified directory\n2. Capture any error output from the command\n3. If the command fails, return an error indicating the path is not a git repository (including captured stderr)\n4. If the command succeeds, return nil (success)\n\n#### [STATE_AND_CONCURRENCY]\nNo mutable state\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n**External I/O Interactions:**\n\n- **External Process Execution**: `exec.Command(\"git\", \"--no-pager\", \"rev-parse\", \"--is-inside-work-tree\")` spawns the `git` binary as a child process. This is disk I/O (reading/writing filesystem state) and process I/O (stdin/stdout/stderr).\n- **Stderr Capture**: `cmd.Stderr = \u0026stderr` redirects the child process's stderr to a `bytes.Buffer`, capturing any error output from the `git` command.\n- **Working Directory**: `cmd.Dir = repoRoot` sets the child process's working directory to the provided `repoRoot` path.\n\n**Error Propagation:**\n\n- **Error Return**: If `cmd.Run()` returns an error, the function wraps it with `fmt.Errorf` and returns the wrapped error to the caller. The wrapped error includes the original error and the captured stderr content.\n- **Success Path**: If `cmd.Run()` returns nil, the function returns `nil` (no error).\n- **No Panic/Exception**: No panics or exceptions are used. Errors are propagated via the function's return value.\n- **No Swallowed Errors**: The error from `cmd.Run()` is not swallowed; it is explicitly returned to the caller." }, "main.go": { "sha256": "16ab362b1830374f0306a1990dba8d51da887c1ad126f311c03d242933d2a516", - "facts": "#### [API_SIGNATURES]\n# Public API Surface — `main.go`\n\n## Functions\n\n- **`main()`** `func() error` \n - Entry point of the program. Delegates to `cmd.RootCmd.Execute()`. On error, prints a message to stderr and exits with code 1.\n - Returns: `error` (via `fmt.Fprintf`/`os.Exit` path — swallowed into exit)\n\n## Referenced External Types \u0026 Methods\n\n- **`RootCmd`** — type from package `github.com/arrase/code-reducer/cmd`. \n - Method: **`Execute()`** — invoked as `cmd.RootCmd.Execute()`.\n\n#### [BUSINESS_LOGIC]\n## Core Purpose \u0026 Domain Rules\n\n### Primary Domain Problem Solved\nThis code provides only an **entry-point shell** for a command-line tool. No domain logic is implemented in this file — all processing happens inside the `cmd` package (`github.com/arrase/code-reducer/cmd`). The entry point itself just:\n\n1. Delegates execution to `cmd.RootCmd.Execute()`\n2. Swallows any error by printing it to stderr and exiting with code 1\n3. Returns success (exit code 0) if no error occurred\n\n### High-Level Algorithm Steps\nThere are none visible in this file. The algorithm is entirely externalized into the imported command package.\n\n#### [STATE_AND_CONCURRENCY]\nNo mutable state\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n**External I/O:**\n- `os.Stderr` — writes formatted error message to standard error stream when execution fails. This is an external side effect (terminal output).\n- `os.Exit(1)` — terminates the process with exit code 1. External system interaction.\n- The third-party package (`github.com/arrase/code-reducer/cmd`) is called via `cmd.RootCmd.Execute()`. No internal source text available to determine what I/O it performs (file writes, network calls, DB queries, etc.). Cannot confirm or deny external effects from this package.\n\n**Error Propagation:**\n- Errors returned by `cmd.RootCmd.Execute()` are propagated as a single-line formatted string (`fmt.Fprintf`) printed to stderr.\n- No retry logic, no fallback paths, no partial recovery — error is terminal.\n- If `Execute()` returns nil (no error), the program exits cleanly with code 0 (default). No explicit call to `os.Exit(0)` but Go's default exit behavior applies." - }, - "mkdocs.yml": { - "sha256": "1e3b3dd1b88dd6996bac64233f41ab78427f0fe86a7090fee60182ab4feaaa2f", - "facts": "#### [API_SIGNATURES]\nNo public API elements (classes, functions, methods, types) were found in the provided source.\n\n#### [BUSINESS_LOGIC]\n**Primary Domain Problem**\nGenerating developer-friendly code wikis from codebases by utilizing local Large Language Models (LLMs) via Ollama.\n\n**High-level Algorithm Steps**\n1. Process codebases using a Map-Reduce engine.\n2. Manage updates through incremental processing and caching.\n3. Execute operations within a security sandbox and using concurrency.\n4. Interface with local LLMs via Ollama.\n\n#### [STATE_AND_CONCURRENCY]\nNo mutable state\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n### External I/O\n* **Network**: \n * Fetches JavaScript from `cdn.jsdelivr.net`.\n * Fetches CSS from `cdnjs.cloudflare.com`.\n\n### Error Propagation\n* No error propagation mechanisms are defined in the configuration file." - }, - "sonar-project.properties": { - "sha256": "cdc6154dcbff4c6c4c342b8e3a3b382e68b4ab596fa44790bae5e83591edd8f7", - "facts": "#### [API_SIGNATURES]\nNo public API elements (classes, functions, methods, or types) were found in the provided file.\n\n#### [BUSINESS_LOGIC]\n**Primary Domain Problem**\nConfigures static analysis and code coverage parameters for a Sonar scanner to facilitate automated code quality reporting for a software project.\n\n**High-level Algorithm Steps**\n1. Define project identification and character encoding.\n2. Designate root directories for source code and test files.\n3. Filter source code by excluding test files, dependencies, documentation, and auxiliary assets.\n4. Filter the test suite by including specific file patterns and excluding dependencies.\n5. Locate the code coverage report for processing.\n\n#### [STATE_AND_CONCURRENCY]\nNo mutable state\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n**External I/O**\n* **Disk:** Reads source files from the current directory (`.`) and coverage reports from `coverage.out`.\n\n**Error Propagation**\n* No error propagation logic is defined in this configuration file." + "facts": "#### [BUSINESS_LOGIC]\nPrimary domain problem: None.\nHigh-level algorithm steps: None.\n\n#### [STATE_AND_CONCURRENCY]\nNo mutable state\n\n#### [ERRORS_AND_SIDE_EFFECTS]\n**External I/O:** None. This file contains no direct network, disk, database, or API interactions. All external behavior is delegated to the `cmd` package via `cmd.RootCmd.Execute()`.\n\n**Error Propagation:** Errors from `cmd.RootCmd.Execute()` are propagated as return values. If non-nil, the error is printed to `os.Stderr` and the process exits with code 1. No panics or unhandled errors are present." } }, "modules": { - ".": "# Code-Reducer System Architecture Reference\n\n## System Overview\nCode-Reducer is a command-line tool designed to generate structured developer documentation (wikis) from codebases by utilizing local Large Language Models (LLMs) via Ollama. The system employs a Map-Reduce engine to perform hierarchical synthesis, transitioning from individual file analysis to high-level architectural documentation. It supports incremental updates by tracking file state via SHA256 fingerprints.\n\n---\n\n## CLI Orchestration (`cmd` module)\n\n### Responsibility\nThe `cmd` module acts as the orchestration layer, managing the CLI lifecycle, environment validation, and dispatching execution to the underlying engine. It handles user interaction for initial configuration and enforces state transitions between project initialization and incremental updates.\n\n### Data Flow\n1. **Input Acquisition**: Captures CLI arguments and interactive user input via `os.Stdin`.\n2. **Environment Validation**: Verifies the existence of a git repository and identifies the working directory.\n3. **State Assessment**: Evaluates the current project state (e.g., via `engine.IsInitialized`) to enforce valid modes (`ModeInit` vs. `ModeUpdate`).\n4. **Configuration Resolution**: Loads and merges settings from disk and environmental sources.\n5. **Execution Dispatch**: Passes the validated `config.Config` and execution `mode` to the `engine` package.\n6. **Output Stream**: Directs status updates and errors to `stdout` and `stderr`.\n\n### Core Components\n* **`RootCmd`**: The primary `cobra.Command` entry point.\n* **`executeCommand`**: The central controller that orchestrates directory resolution, git verification, and configuration loading.\n* **`RunSetupFlow`**: An interactive process that captures LLM Model IDs, Ollama endpoints, context sizes, and documentation directory paths.\n* **Subcommands**:\n * `init`: Triggers a full repository scan and initial Markdown generation.\n * `update`: Triggers incremental analysis by comparing current file states against the last documented git commit.\n\n---\n\n## Configuration Management (`internal/config`)\n\n### Responsibility\nThis module manages the hierarchical resolution, validation, and persistence of system settings required for the LLM pipeline.\n\n### Data Flow\n1. **Ingestion**: Aggregates data from `.code-reducer.yaml`, environment variables, and CLI flags.\n2. **Resolution**: Resolves values based on the following precedence: CLI flags $\\rightarrow$ Environment variables $\\rightarrow$ Configuration file $\\rightarrow$ System defaults.\n3. **Validation**: Ensures numerical parameters (e.g., `OllamaNumCtx`) are integers greater than zero.\n4. **Persistence**: Serializes the `Config` struct to YAML and performs an atomic write using a temporary file and rename pattern.\n\n### Data Structures\n* **`Config`**: The primary schema containing `ModelID`, `OllamaBaseURL`, `OllamaNumCtx`, `DocsDir`, and prompt templates for different synthesis stages (Module, Architecture, File Fact Consolidation).\n* **`ExtractionStep`**: Defines specific phases of the LLM extraction pipeline, mapping a `Name` to a specific `Prompt`.\n\n---\n\n## Map-Reduce Documentation Engine (`internal/engine`)\n\n### Responsibility\nThe engine performs the core logic of transforming source code into structured documentation through a hierarchical reduction process. It manages file discovery, change detection, and recursive LLM-based content reduction.\n\n### Data Flow\n1. **Acquisition**: The `Runner` secures a repository lock.\n2. **Discovery**: Files are mapped into a hierarchical `DirNode` tree structure.\n3. **Change Detection**: The `Orchestrator` compares current file SHA256 hashes against the `MetadataCache`. \"Affected\" status propagates up the `DirNode` tree.\n4. **Hierarchical Synthesis**:\n * **Leaf Processing**: Files are processed via `Chunking`. If content exceeds the context window (using a 0.75 `contextWindowAllocRatio`), recursive reduction is applied.\n * **Aggregation**: Extracted information is aggregated from the file level up to the module and directory levels.\n5. **Persistence**: Synthesized Markdown and `MetadataCache` (JSON) are written to the `docsDir`.\n\n### Core Components\n* **`Synthesizer`**: Traverses the `DirNode` tree to decide between using cached summaries or triggering new LLM extractions.\n* **`Chunking`**: Implements recursive reduction for oversized segments. Recursion terminates if the output size is $\\ge$ 95% of the input.\n* **`MetadataCache`**: Persists SHA256 fingerprints to facilitate incremental updates.\n* **`LLM Client`**: A synchronous HTTP client communicating via the `api/chat` protocol.\n\n---\n\n## External Tooling Configurations\n\n### `mkdocs.yml`\nDefines the deployment parameters for the generated documentation wiki.\n* **Domain Logic**: Orchestrates the Map-Reduce engine and local LLM interaction.\n* **External I/O**: Fetches frontend assets (JS/CSS) from `cdn.jsdelivr.net` and `cdnjs.cloudflare.com`.\n\n### `sonar-project.properties`\nConfigures static analysis and code coverage reporting.\n* **Filtering Logic**: Excludes test files, dependencies, and auxiliary assets from source analysis.\n* **I/O**: Reads from the current directory and the `coverage.out` report.", - "cmd": "# Module: `cmd`\n\n## Module Overview\nThe `cmd` module serves as the command-line interface (CLI) orchestration layer. It manages the lifecycle of project documentation generation by coordinating user input, configuration persistence, git repository validation, and the execution of the underlying documentation engine.\n\n## Data Flow\n1. **Input Acquisition**: CLI arguments are parsed via `RootCmd`. User-provided flags (`--model-id`, `--num-ctx`) and interactive input (`os.Stdin`) are captured.\n2. **Environment Validation**: The module verifies the presence of a git repository and determines the current working directory via `os.Getwd`.\n3. **State Assessment**: The module checks for existing configuration files and determines if the project is in an initialized state to validate the requested operation (`init` vs. `update`).\n4. **Configuration Resolution**: Configuration is loaded from disk, merged with defaults, and optionally updated via an interactive setup flow.\n5. **Execution Dispatch**: The validated configuration and mode are passed to the `engine` package to perform documentation generation or incremental updates.\n6. **Output Stream**: Results, status updates, and errors are streamed to `stdout` and `stderr`.\n\n## Core Components\n\n### Command Orchestration (`root.go`)\nThe central logic for the CLI lifecycle is contained within this file.\n\n* **`RootCmd`** (`*cobra.Command`): The entry point for the CLI application.\n* **`executeCommand(mode engine.Mode) error`**: The primary controller. It performs the following sequence:\n * Resolves the working directory.\n * Verifies the git repository status.\n * Triggers `checkAndRunSetup` if configuration is missing and `os.Stdin` is a TTY.\n * Resolves the final configuration.\n * Validates that the current project state matches the requested `engine.Mode`.\n * Dispatches execution to `runEngine`.\n* **`checkAndRunSetup(repoRoot string) error`**: Evaluates if `RunSetupFlow` is required based on the existence of configuration files and terminal availability.\n* **`checkInitStatus(repoRoot, docsDir string, mode engine.Mode) error`**: Enforces state transition rules:\n * `ModeInit` is prohibited if the project is already initialized.\n * `ModeUpdate` is prohibited if the project is not yet initialized.\n* **`runEngine(repoRoot string, cfg *config.Config, mode engine.Mode) error`**: Orchestrates engine execution with signal handling for `os.Interrupt` and `syscall.SIGTERM`.\n\n### Command Definitions (`init.go`, `update.go`)\nThese files act as thin wrappers that map CLI subcommands to specific engine modes.\n\n* **`init` subcommand**: Dispatches `engine.ModeInit` to trigger the initial scanning of the repository and generation of wiki markdown files.\n* **`update` subcommand**: Dispatches `engine.ModeUpdate` to perform incremental updates by comparing current file states against the last documented git commit.\n\n### Configuration Lifecycle (`setup.go`)\nHandles the interactive provisioning of project settings.\n\n* **`RunSetupFlow(repoRoot string) error`**: Manages a stateful, interactive session to collect:\n * LLM Model ID.\n * Ollama Base URL.\n * Ollama Context Size.\n * File/directory ignore patterns.\n * Documentation directory paths.\n* **Persistence**: Finalized settings are consolidated into a `config.Config` object and written to disk via `config.SaveConfig`.\n\n## State and Side Effects\n\n### Mutable State\n* **Global Flags**: `modelIDFlag` and `numCtxFlag` are modified during the `init()` phase and persist for the lifetime of the process.\n* **Command State**: `RootCmd` state is modified via `.SetArgs()` and `.Execute()` during testing.\n\n### External I/O\n* **Filesystem**:\n * Directory/file creation via `os.MkdirAll` and `os.WriteFile`.\n * Configuration persistence via `config.SaveConfig` and `config.LoadConfig`.\n * Git verification via `tools.VerifyGitRepo`.\n * State check via `engine.IsInitialized`.\n* **Standard Streams**:\n * `os.Stdin`: Used for interactive parameter collection.\n * `os.Stdout`: Used for status messages and engine events.\n * `os.Stderr`: Used for error reporting.\n\n### Error Handling\n* **Propagation**: Errors are passed up the call stack via return values.\n* **Wrapping**: `os.Getwd` and `runner.Run` errors are wrapped with context using `%w`.\n* **Swallowed Errors**: \n * `config.LoadConfig` errors are swallowed in `RunSetupFlow` (defaults are returned instead).\n * `strconv.Atoi` errors in `promptContextSize` are swallowed (default values are returned instead).\n * `RootCmd.Execute()` return values are swallowed in `cmd_test.go`.", - "internal": "# Code-Reducer System Architecture Reference\n\n## Subsystem: `internal/config`\n\n### Module Responsibility and Data Flow\nThe `internal/config` package manages the lifecycle, hierarchical resolution, and persistence of configuration settings. It enables the merging of settings from multiple sources into a unified `Config` struct to direct the LLM-based pipeline.\n\n**Data Flow Pipeline:**\n1. **Ingestion**: Aggregation of data from the local filesystem (`.code-reducer.yaml`), environment variables, and CLI arguments.\n2. **Resolution**: Hierarchical merging based on precedence: CLI flags \u003e Environment variables \u003e Configuration file \u003e System defaults.\n3. **Validation**: Numerical parameters (e.g., `OllamaNumCtx`) are validated to ensure they are non-zero and valid integers.\n4. **Output**: Returns a validated `*Config` pointer for LLM prompt construction and execution logic.\n5. **Persistence**: Configuration changes are serialized to YAML and written via an atomic rename pattern.\n\n### Data Structures\n\n#### `Config`\nThe primary operational schema for the system.\n\n| Field | Type | Description |\n|---|---|---|\n| `ModelID` | `string` | Identifier for the LLM model. |\n| `OllamaBaseURL` | `string` | Base endpoint for the local Ollama API. |\n| `OllamaNumCtx` | `int` | Context window size for the LLM. |\n| `DocsDir` | `string` | Target directory for generated documentation. |\n| `SystemPrompt` | `string` | Primary system message for the LLM. |\n| `ModuleSynthesisPrompt` | `string` | Prompt template for module-level synthesis. |\n| `ArchitecturePrompt` | `string` | Prompt template for architectural analysis. |\n| `FileFactConsolidationPrompt` | `string` | Prompt template for consolidating file facts. |\n| `ExtractionSteps` | `[]ExtractionStep` | Sequence of extraction phases for the pipeline. |\n| `Ignore` | `[]string` | Paths or patterns excluded from analysis. |\n\n#### `ExtractionStep`\nDefines a specific phase in the multi-step LLM extraction process.\n\n| Field | Type | Description |\n|---|---|---|\n| `Name` | `string` | Identifier for the extraction phase. |\n| `Prompt` | `string` | Specific prompt utilized for this phase. |\n\n### Hierarchical Resolution and Validation\nThe `ResolveConfig` function determines the final state of the `Config` instance using the following precedence:\n1. **Command-line Flags**: `modelIDFlag`, `numCtxFlag`.\n2. **Environment Variables**: `CODE_REDUCER_MODEL_ID`, `OLLAMA_BASE_URL`, `OLLAMA_NUM_CTX`.\n3. **Configuration File**: Values from `.code-reducer.yaml`.\n4. **System Defaults**: Hardcoded values.\n\n**Constraints:**\n* **Numerical Validation**: `OllamaNumCtx` must be a valid integer and greater than zero. Non-zero/non-integer environment variables result in an error.\n* **Deduplication**: The `Ignore` slice is deduplicated during resolution.\n* **Default Logic**: If `ExtractionSteps` is empty in the config file, the system defaults to the `DefaultExtractionSteps` sequence.\n\n### I/O and Persistence\n* **`ConfigExists`**: Employs `os.Stat` to check for the presence of the configuration file. All errors are swallowed; returns `false` on error.\n* **`LoadConfig`**: Utilizes `os.ReadFile` and `yaml.Unmarshal`. Errors are wrapped and returned.\n* **`SaveConfig`**: Implements an atomic write pattern:\n 1. `yaml.Marshal` the struct to bytes.\n 2. `os.CreateTemp` a temporary file in the target directory.\n 3. Write data and invoke `os.Sync` for durability.\n 4. `os.Chmod` to set permissions.\n 5. `os.Rename` to atomically replace the target file.\n * **Note**: The deferred `os.Remove` does not contain a `recover()`.\n\n---\n\n## Subsystem: `internal/engine`\n\n### Module Responsibility and Data Flow\nThe `internal/engine` module is an orchestration engine using a hierarchical Map-Reduce pattern to transform source code into structured documentation via LLMs. It supports `ModeInit` (initial generation) and `ModeUpdate` (incremental updates).\n\n**Data Flow Lifecycle:**\n1. **Initialization**: The `Runner` acquires an exclusive repository lock via `security.AcquireLock`.\n2. **Discovery**: The `Tree` logic converts file paths into a hierarchical `DirNode` structure.\n3. **Change Detection**: The `Orchestrator` compares SHA256 hashes against `MetadataCache`. Changes propagate upward through the `DirNode` tree to mark nodes as \"affected.\"\n4. **Hierarchical Synthesis**: \n * **Leaf Processing**: Files undergo `Chunking` (recursive reduction if context limits are exceeded) followed by LLM extraction.\n * **Aggregation**: Results aggregate from file $\\rightarrow$ module $\\rightarrow$ directory levels.\n5. **Persistence**: Writes synthesized Markdown and the `MetadataCache` (JSON) to the `docsDir`.\n\n### Core Components\n\n#### Orchestration and Execution\n* **Runner (`runner.go`)**: Entry point; manages execution lifecycle and repository locking.\n* **Orchestrator (`orchestrator.go`)**: Manages cache invalidation, directory pruning, and global (root-level) documentation regeneration.\n* **Synthesizer (`synthesize.go`)**: Traverses the `DirNode` tree; decides between cached summaries or new LLM extractions based on \"affected\" status.\n\n#### Data Processing\n* **Chunking and Reduction (`chunking.go`)**: Implements recursive reduction for oversized items using expansion (overlapping segments), batching, and consolidation. Recursion terminates if output is $\\ge 95\\%$ of input.\n* **Markdown Processing (`markdown.go`)**: Strips triple-backtick or JSON fences from LLM output to extract raw content.\n* **Metadata Cache (`cache.go`)**: Persists SHA256 fingerprints. Only version `1` is supported; version mismatches trigger a clean state.\n* **Tree and Change Analysis (`tree.go`)**: Tracks file status (`StatusAdded`, `StatusModified`, `StatusDeleted`) and propagates impact up the directory hierarchy.\n\n#### Infrastructure\n* **LLM Client (`client.go`)**: Synchronous HTTP client for the `api/chat` protocol. Performs no retry logic or circuit breaking.\n* **Constants (`constants.go`)**:\n * `defaultHTTPTimeout`: 10 minutes.\n * `contextWindowAllocRatio`: 0.75 (75% for content, 25% for overhead).\n * `defaultChunkOverlap`: 800 tokens.\n * `maxErrorBodyBytes`: 1 KB.\n* **Utilities (`utils.go", - "internal/config": "# Package `internal/config`\n\n## Module Responsibility\nThe `internal/config` package manages the lifecycle, hierarchical resolution, and persistence of configuration settings for the Code-Reducer system. It provides the mechanism to merge settings from multiple sources—including command-line flags, environment variables, and configuration files—into a unified `Config` struct. This configuration directs an LLM-based pipeline through specific extraction and synthesis phases.\n\n### Data Flow\n1. **Ingestion**: The system gathers data from the local filesystem (via `.code-reducer.yaml`), environment variables, and CLI arguments.\n2. **Resolution**: A hierarchical merge process applies overrides based on a defined priority: CLI flags \u003e Environment variables \u003e Configuration file values \u003e System defaults.\n3. **Validation**: Numerical parameters, such as context size, undergo validation to ensure they meet non-zero requirements.\n4. **Output**: A validated `*Config` pointer is returned for use in LLM prompt construction and execution logic.\n5. **Persistence**: Configuration changes are serialized to YAML and written to the filesystem using an atomic rename pattern to ensure integrity.\n\n## Data Structures\n\n### `Config`\nThe primary schema representing the system's operational parameters.\n\n| Field | Type | Description |\n|---|---|---|\n| `ModelID` | `string` | The identifier for the LLM model. |\n| `OllamaBaseURL` | `string` | The base endpoint for the local Ollama API. |\n| `OllamaNumCtx` | `int` | The context window size for the LLM. |\n| `DocsDir` | `string` | The target directory for generated documentation. |\n| `SystemPrompt` | `string` | The primary system message for the LLM. |\n| `ModuleSynthesisPrompt` | `string` | Prompt template for module-level synthesis. |\n| `ArchitecturePrompt` | `string` | Prompt template for architectural analysis. |\n| `FileFactConsolidationPrompt` | `string` | Prompt template for consolidating file facts. |\n| `ExtractionSteps` | `[]ExtractionStep` | A sequence of extraction phases for the analysis pipeline. |\n| `Ignore` | `[]string` | A list of paths or patterns to be excluded from analysis. |\n\n### `ExtractionStep`\nDefines a single phase in the multi-step LLM extraction process.\n\n| Field | Type | Description |\n|---|---|---|\n| `Name` | `string` | The identifier for the extraction phase. |\n| `Prompt` | `string` | The specific prompt utilized for this phase. |\n\n## Hierarchical Resolution\n\nThe `ResolveConfig` function implements the precedence logic to determine the final state of a `Config` instance.\n\n### Resolution Precedence\nFor each parameter, the first non-default/non-empty value found in the following order is selected:\n1. **Command-line Flags** (e.g., `modelIDFlag`, `numCtxFlag`).\n2. **Environment Variables** (defined by `CodeReducerModelIDEnvKey`, `OllamaNumCtxEnvKey`, and `OllamaBaseURLEnvKey`).\n3. **Configuration File** (values stored in `.code-reducer.yaml`).\n4. **System Defaults** (hardcoded values).\n\n### Numerical Validation\nThe resolution of `OllamaNumCtx` requires that the value—whether sourced from a flag, environment variable, or file—is a valid integer and greater than zero. If an environment variable is non-empty but fails `strconv.Atoi` or is $\\le 0$, `ResolveConfig` returns an error.\n\n### List and Default Initialization\n* **Ignore List**: The `Ignore` slice is deduplicated during resolution.\n* **Extraction Steps**: If the configuration file provides an empty list for `ExtractionSteps`, the system defaults to the `DefaultExtractionSteps` slice.\n\n## I/O and Persistence\n\n### Filesystem Operations\nThe module utilizes a `ConfigFileName` (`.code-reducer.yaml`) within a specified directory to manage persistence.\n\n* **`ConfigExists`**: Checks for the existence of the configuration file using `os.Stat`. All errors encountered during this operation are swallowed, and the function returns `false`.\n* **`LoadConfig`**: Reads the configuration file via `os.ReadFile` and unmarshals the content via `yaml.Unmarshal`. Errors from file reading or YAML parsing are wrapped and returned to the caller.\n* **`SaveConfig`**: Implements an atomic write pattern to prevent file corruption:\n 1. Serializes the `Config` struct to YAML bytes via `yaml.Marshal`.\n 2. Creates a temporary file in the target directory using `os.CreateTemp`.\n 3. Writes the data and ensures durability via `os.Sync`.\n 4. Sets file permissions via `os.Chmod`.\n 5. Closes the file and uses `os.Rename` to atomically replace the target file.\n * **Note**: The deferred cleanup (via `os.Remove`) in `SaveConfig` does not contain `recover()`; panics during the deferred execution are not handled.\n\n## Constants and Defaults\n\n### Environment Variable Keys\n| Key | Purpose |\n|---|---|\n| `CODE_REDUCER_MODEL_ID` | Overrides the LLM `ModelID`. |\n| `OLLAMA_BASE_URL` | Overrides the `OllamaBaseURL`. |\n| `OLLAMA_NUM_CTX` | Overrides the `OllamaNumCtx`. |\n\n### Default Extraction Pipeline (`DefaultExtractionSteps`)\nWhen no custom extraction steps are provided, the system executes these four stages:\n1. **`API_SIGNATURES`**: Extracts public types, functions, and methods.\n2. **`BUSINESS_LOGIC`**: Synthesizes the primary domain problem and high-level algorithmic steps.\n3. **`STATE_AND_CONCURRENCY`**: Identifies mutable state and synchronization mechanisms. Returns \"No mutable state\" if none are found.\n4. **`ERRORS_AND_SIDE_EFFECTS`**: Details external I/O interactions and error propagation. Returns \"No external side effects\" if no I/O is detected.", - "internal/engine": "# Internal Engine Architecture Documentation\n\n## Module Responsibility and Data Flow\n\nThe `internal/engine` module is an automated documentation orchestration engine that employs a hierarchical Map-Reduce pattern to transform repository source code into structured technical documentation using Large Language Models (LLMs). It facilitates both initial documentation generation (`ModeInit`) and incremental updates (`ModeUpdate`) by implementing a change-detection pipeline based on file integrity hashes and directory-level impact analysis.\n\n### Data Flow Lifecycle\n1. **Initialization \u0026 Locking**: The `Runner` acquires an exclusive repository lock and ensures the `.gitignore` contains the lockfile.\n2. **Discovery \u0026 Tree Construction**: `Tree` logic converts a flat list of file paths into a hierarchical `DirNode` structure.\n3. **Change Detection**: The `Orchestrator` compares current file SHA256 hashes against the `MetadataCache`. Changes are mapped to specific `FileChange` objects and propagated upward through the `DirNode` tree to identify \"affected\" directory nodes.\n4. **Hierarchical Synthesis**:\n * The `Synthesizer` performs a bottom-up traversal of the affected tree branches.\n * **Leaf Processing**: Files are read, content is processed via `Chunking` (recursive reduction if context limits are exceeded), and LLM extraction is performed.\n * **Aggregation**: Results from files and subdirectories are aggregated and passed to the LLM for higher-level synthesis.\n5. **Persistence**: Synthesized Markdown documentation and the updated `MetadataCache` are written to the filesystem.\n\n---\n\n## Orchestration and Execution Control\n\n### Runner (`runner.go`)\nServes as the primary entry point for the module. It manages the execution lifecycle, handles repository-level synchronization via `security.AcquireLock`, and delegates core logic to the `Orchestrator`. It supports two operational modes: `ModeInit` and `ModeUpdate`.\n\n### Orchestrator (`orchestrator.go`)\nCoordinates the end-to-end pipeline. It manages the transition from file discovery to hierarchical synthesis. It handles the logic for cache invalidation, directory pruning for deleted files, and determines whether global documentation (e.g., `architecture.md`) requires regeneration based on the impact of changes detected in the root directory.\n\n---\n\n## Hierarchical Synthesis and Data Processing\n\n### Synthesizer (`synthesize.go`)\nImplements the bottom-up reduction algorithm. It traverses the `DirNode` tree, determining for each node whether to use cached summaries or perform new LLM-based extractions based on the \"affected\" status. It manages the recursive aggregation of file-level facts into module-level and directory-level summaries.\n\n### Chunking and Reduction (`chunking.go`)\nAddresses LLM context window constraints through a recursive reduction strategy:\n* **Expansion**: Splits oversized items into overlapping segments.\n* **Batching**: Groups items to optimize context window utilization.\n* **Recursive Reduction**: Applies LLM-based consolidation to batches.\n* **Convergence Check**: Stops recursion if a reduction step fails to decrease the input size by more than 5% (i.e., output $\\ge$ 95% of input).\n\n### Markdown Processing (`markdown.go`)\nProvides utilities for cleaning LLM output, specifically stripping surrounding triple-backtick Markdown or JSON fences to extract raw content.\n\n---\n\n## State, Change Detection, and Cache Management\n\n### Metadata Cache (`cache.go`)\nProvides a persistence layer for tracking extracted documentation facts and file integrity.\n* **Versioning**: Only version `1` is accepted; version mismatches trigger a clean cache state.\n* **Integrity**: Uses SHA256 fingerprints of raw file content at the virtual path relative to the repository root.\n* **Persistence**: Serializes state to a JSON file in the specified `docsDir`.\n\n### Tree and Change Analysis (`tree.go`)\nImplements dependency impact analysis within the directory hierarchy.\n* **Status Tracking**: Tracks files as `StatusAdded`, `StatusModified`, or `StatusDeleted`.\n* **Upward Propagation**: If a child node (file or directory) is marked as affected, its status is recursively propagated to all ancestor `DirNode` objects.\n* **Impact Detection**: Identifies affected nodes based on missing cache entries, modified hashes, or missing documentation files.\n\n---\n\n## Infrastructure and Communication\n\n### LLM Client (`client.go`)\nAn HTTP client targeting the Ollama `api/chat` protocol. \n* **Interface**: Abstraction via `llmCaller` allows for decoupling the engine from the specific transport.\n* **Behavior**: Executes synchronous POST requests. It performs no retry logic on transport-level failures and contains no circuit-breaking mechanisms.\n\n### Configuration and Constants (`constants.go`)\nDefines operational parameters, including:\n* **Timeouts**: `defaultHTTPTimeout` (10 minutes).\n* **Context Management**: A `contextWindowAllocRatio` of 0.75 reserves 75% of the context window for primary content, leaving 25% for metadata/overhead.\n* **Chunking**: `defaultChunkOverlap` (800 tokens).\n* **Limits**: `maxErrorBodyBytes` (1 KB) caps error response parsing.\n\n### Utilities (`utils.go`)\nProvides filesystem path sanitization for generating `README.md` files and a nil-safe event logging adapter.", - "internal/security": "# Module: `internal/security` — Path Traversal Prevention and Process-Level Locking\n\n## Module Responsibility\n\nThis module provides two security primitives scoped to a repository root:\n\n1. **Path traversal prevention** via `SafeResolve`, which validates that user-supplied paths resolve within the repository boundary, resolving symlinks on existing ancestor parts before comparison.\n2. **Cross-process lock management** via `SimpleLock`/`AcquireLock`/`Unlock`, which ensures only one process holds exclusive access to a protected resource at any given time through an atomic file-based lock mechanism.\n\nBoth primitives operate against the local filesystem; no network, database, or remote API interactions occur within production code. All errors are returned to callers and propagated via Go's `error` interface; panics are not raised by this module.\n\n## Data Flow Overview\n\n1. Callers invoke `SafeResolve(repoRoot, inputPath)` with a candidate path string and an anchor directory; the function returns either a resolved absolute path confined within `repoRoot` or an error (typically `ErrPathTraversal`).\n2. Callers invoke `AcquireLock(repoRoot)` to obtain a lock handle; if contention is detected—either via OS-level atomic failure of `O_EXCL` or presence of a stale lockfile—the function returns an error wrapping the sentinel `ErrLockHeld`.\n3. When release is needed, callers call `Unlock()` on the returned `*SimpleLock`; this method is idempotent and thread-safe with respect to itself, closing the file descriptor and removing the lockfile from disk under a mutex guard.\n\n## Error Definitions — `errors.go`\n\nThe module declares two sentinel error variables used as context markers during error propagation:\n\n| Sentinel | Declaration Scope | Usage Context |\n|---|---|---|\n| `ErrPathTraversal` | package-level in `internal/security/errors.go` | Returned by `SafeResolve` when the resolved path escapes outside `repoRoot`. The original traversal signal is swallowed and replaced with a formatted message carrying `inputPath` as context. |\n| `ErrLockHeld` | package-level in `internal/security/errors.go` | Returned by `AcquireLock` when another process holds the lock or a stale lockfile persists; returned via `%w` wrapping of the underlying OS error with descriptive context. |\n\nNo functions, imports beyond `errors`, or runtime operations exist in this file. Error propagation from these sentinels (e.g., further wrapping via `fmt.Errorf`) occurs elsewhere, not here.\n\n## Path Resolution — `SafeResolve`\n\n### Signature\n\n```go\nfunc SafeResolve(repoRoot, inputPath string) (string, error)\n```\n\n### Responsibility\n\nResolves a candidate path against an anchor root while preventing escape through symlinks and directory traversal. The function returns the cleaned absolute path if it remains strictly inside the resolved repository boundary; otherwise it returns `ErrPathTraversal`.\n\n### Algorithm Steps\n\n1. Compute the absolute root directory from `repoRoot` via `filepath.Abs`, wrapping any resulting error with `%w`.\n2. Resolve symlinks on the absolute root using `filepath.EvalSymlinks(absRoot)`, again wrapping errors with `%w`.\n3. Walk up from the joined path (`absRoot + inputPath`) until a physically existing ancestor is found via repeated `os.Lstat(current)` calls; each Lstat failure that is not \"not exist\" is wrapped and returned immediately.\n4. Resolve symlinks on the first physically existing ancestor via `filepath.EvalSymlinks(current)`, wrapping errors with `%w`.\n5. Reconstruct the full target path from the resolved ancestor plus all previously-skipped components.\n6. Verify that the reconstructed path remains inside the resolved root; reject if it escapes by returning a wrapped error using `ErrPathTraversal` with `inputPath` as context.\n\n### External I/O Operations\n\n| Operation | Function | Purpose |\n|---|---|---|\n| Absolute path resolution from relative input | `filepath.Abs(repoRoot)` | Establishes the canonical root baseline for comparison |\n| Symlink resolution on root | `filepath.EvalSymlinks(absRoot)` | Prevents symlink-based escape at the boundary |\n| Ancestor existence check | `os.Lstat(current)` loop | Walks up the path tree stopping at first physically existing parent |\n| Symlink resolution on ancestor | `filepath.EvalSymlinks(current)` | Prevents symlink-based escape mid-path |\n\n## Lock Acquisition and Release — `SimpleLock`\n\n### Types\n\n#### `SimpleLock` Struct\n\n| Field | Type | Mutability | Notes |\n|---|---|---|---|\n| `lockPath` | `string` | Modified once during acquisition, then read-only thereafter. Not mutated by any other method. | Absolute path of the lockfile within the repository root. |\n| `file` | `*os.File` | Modified once during acquisition. Closed during unlock and removed from disk afterward. | File descriptor holding the exclusive lock; opened with `O_WRONLY\\|O_CREATE\\|O_EXCL`. |\n| `mu` | `sync.Mutex` | Read via lock/unlock primitives only. Field itself is never reassigned after struct initialization. | Protects the close-and-remove sequence in `Unlock()`, making unlock atomic with respect to itself. |\n| `closed` | `bool` | Set to `true` during unlock; read inside unlock for idempotency check (`if l.closed { return nil }`). | Tracks release state so subsequent calls return immediately without further I/O. |\n\n#### Methods\n\n##### `AcquireLock(repoRoot string) (*SimpleLock, error)`\n\n**Responsibility**: Acquires an exclusive file-based lock within the provided repository root. Uses O_EXCL to ensure atomicity; failure implies another process holds the lock or a stale lockfile exists.\n\n**Algorithm Steps**:\n1. Calls `SafeResolve(repoRoot, LockFileName)` to obtain the canonical lock file path inside the repo root. If this fails, the error propagates directly without wrapping.\n2. Opens the lockfile with `os.OpenFile(lockPath, O_WRONLY\\|O_CREATE\\|O_EXCL, 0644)`. The OS-level atomicity of O_EXCL means failure indicates another writer holds the file or a stale lock persists; this is treated as an error condition requiring manual cleanup by the caller.\n3. Writes the current process PID into the lockfile via `f.Write([]byte(fmt.Sprintf(\"%d\\n\", os.Getpid())))` for identification/inspection purposes. If this write fails, the method closes the file and removes it before returning a wrapped error.\n\n**Error Propagation**:\n- Direct propagation when `SafeResolve` fails (no wrapping).\n- Wraps with `%w` using `ErrLockHeld` plus context string describing stale lockfile when the OS reports `os.IsExist`.\n- On write failure after successful OpenFile: closes file and removes it, then returns a wrapped error that swallows close/remove errors into a single formatted message—only the original write error is surfaced.\n\n##### `Unlock() error` (method on `*SimpleLock`)\n\n**Responsibility**: Releases the lock by closing the file descriptor and removing the lockfile from disk. Idempotent and thread-safe with respect to itself.\n\n**Algorithm Steps**:\n1. Acquires `l.mu.Lock()` at entry; releases via `defer mu.Unlock()`.\n2. Checks `if l.closed { return nil }`; if already closed, returns immediately without further I/O.\n3. Closes the file descriptor (`l.file.Close()`). If this fails, the error is swallowed when reporting removal errors—only surfaces the remove error if it subsequently fails.\n4. Attempts to remove the lockfile from disk via `os.Remove(l.lockPath)`.\n\n**Thread Safety**: The struct owns its own mutex; concurrent calls to `Unlock` on the same instance serialize through `mu.Lock()/defer mu.Unlock()`, making the operation atomic with respect to itself. Concurrent acquisition of different instances each creates independent `SimpleLock` objects with separate mutexes—no contention between instances is modeled or guaranteed by this code.\n\n## Test Coverage — `security_test.go`\n\n### Functions Under Test\n\n| Function | Parameters | Return Values |\n|---|---|---|\n| `SafeResolve(repoRoot, inputPath)` | repository root path; input file/directory path to resolve | resolved absolute path (string); error |\n| `AcquireLock(repoRoot)` | repository root path | lock object (`*SimpleLock`); error |\n\n### Test Fixture Operations\n\nAll operations below target the local filesystem via Go's standard library:\n\n| Operation | Function | Purpose | Error Handling |\n|---|---|---|---|\n| Directory creation for test fixtures | `os.MkdirAll()` | Sets up `src/sub/` directories | Propagated to test framework via `t.Fatal(err)` — terminates immediately, no recovery. |\n| File writing for traversal tests | `os.WriteFile()` | Creates `file.txt`, `..config` | Same as above; propagated to `t.Fatal(err)`. |\n| Symlink creation (conditional) | `os.Symlink()` | Creates a symlink pointing outside the repo root | Swallowed and replaced with `t.Skip(\"Symlinks not supported on this OS/filesystem\")`; test continues if operation fails. |\n\n### Test Assertions\n\n- `security.SafeResolve(repoRoot, inputPath)` returns an error to its caller; tests verify both success and failure paths via a `wantErr` flag comparison (`t.Errorf(...)`).\n- `security.AcquireLock(repoRoot)` returns `(lock, err)`. A second concurrent call failing with `AcquireLock() should have failed when lock is already held` confirms the production code signals contention via error return.\n\n## Concurrency and Synchronization Summary\n\n- No package-level variables are declared anywhere in this module; all mutable state is encapsulated within individual `SimpleLock` instances.\n- `Unlock()` self-synchronizes via `mu.Lock()/defer mu.Unlock()`, making it safe for concurrent calls on the same instance.\n- Concurrent calls to `AcquireLock` on different instances each create their own independent `SimpleLock` with separate mutexes—no contention between instances is modeled or guaranteed by this code.\n- The file-level lock (the `.code-reducer.lock` file itself) provides OS-level mutual exclusion, which complements the in-memory `mu`.\n- No locks/mutexes are used within the test file itself; each `Test*` function runs independently in a single goroutine.", - "internal/tools": "# Module: `internal/tools` — Repository Analysis Utilities\n\n## Responsibility \u0026 Data Flow\n\nThe `internal/tools` package provides two complementary capabilities for analyzing a code repository's structure: (1) safe file I/O operations with TOCTOU protection and atomic writes, and (2) Git command abstraction for process execution. All functions operate against a local filesystem only; no network calls, database interactions, or external API invocations occur.\n\nThe module's entry point is the `repoRoot` string passed to every public function. Data flows from caller → `internal/tools` package functions → OS subsystems (POSIX file system for I/O operations, git binary via subprocess execution). No mutable state exists within any function scope; all operations are stateless and idempotent with respect to repository content.\n\n---\n\n## File Operations (`file_tools.go`)\n\n### Atomic Write — `WriteFileAtomic`\n\n```go\nfunc WriteFileAtomic(targetPath string, data []byte, perm os.FileMode) error\n```\n\nWrites binary data to `targetPath` using a temp-file + rename pattern that prevents partial or corrupt state if interrupted. The directory is created with mode `0755`. Returns an error on any failure; returns `nil` only after the final rename succeeds.\n\n**Sequence:**\n1. Create temporary file in same directory as target (ensures atomic rename semantics)\n2. Write data to temp file, sync before close for crash safety\n3. Close and flush via defer block with cleanup gated by success flag\n4. Apply `perm` mode via `os.Chmod`\n5. Atomic rename from temp path into final location\n\nErrors at any step are wrapped; deferred cleanup runs on failure paths to prevent partial temp file leakage. No panic/recover anywhere in this function.\n\n### Safe Read — `ReadFileSafely`\n\n```go\nfunc ReadFileSafely(repoRoot string, virtualPath string) ([]byte, error)\n```\n\nResolves the virtual path against `repoRoot`, then reads via `os.ReadFile`. Wraps any read failure with context (\"failed to read file content\"). The path resolution error from `security.SafeResolve` propagates unwrapped.\n\n### Safe Write — `WriteFileSafely`\n\n```go\nfunc WriteFileSafely(repoRoot string, virtualPath string, content []byte) error\n```\n\nDelegates to both `ReadFileSafely` (for path resolution) and `WriteFileAtomic`. Unwrapped `security.SafeResolve` error propagates through. Otherwise defers to `WriteFileAtomic`'s return value.\n\n### Gitignore Integration (`file_tools.go`)\n\n#### Lockfile Recording — `EnsureGitignoreHasLockfile`\n\n```go\nfunc EnsureGitignoreHasLockfile(repoRoot string) error\n```\n\nEnsures `.gitignore` contains a line matching `security.LockFileName`. Reads the existing file, appends the lock entry if absent (adding a newline before it when needed), and writes back atomically. Returns an error on read/write failures. Read errors from `.gitignore` (except `os.IsNotExist`) are wrapped with \"error reading .gitignore\". A non-existent `.gitignore` falls through to the write path, which will fail if the directory does not exist; no special handling for that intermediate state.\n\n#### Loading — `LoadGitignore`\n\n```go\nfunc LoadGitignore(repoRoot string) ([]string, error)\n```\n\nReads `.gitignore` from `repoRoot`, strips comments (`#`) and blank lines, returns remaining patterns as `[]string`. Returns `nil, nil` when no `.gitignore` exists (treated as success). Any other read failure propagates unwrapped. Scanner errors appended via `scanner.Err()` at end — if a scan error occurs mid-stream but lines have already been populated, the partial result still gets returned to the caller.\n\n#### Path Filtering — `ShouldIgnoreFile`\n\n```go\nfunc ShouldIgnoreFile(relPath string, gitIgnore *ignore.GitIgnore) bool\n```\n\nReturns `true` when the given relative path should be ignored: matches via the provided GitIgnore object (if non-nil), contains a dot-prefixed component, or ends with `.egg-info`. No I/O; returns only boolean values. No error path.\n\n#### Discovery — `DiscoverCodeFiles`\n\n```go\nfunc DiscoverCodeFiles(repoRoot string, ignores []string) ([]string, error)\n```\n\nRecursively walks `repoRoot` to collect relative file paths that pass ignore filtering. Skips directories whose names start with `.`, end with `.egg-info`, or match the compiled GitIgnore rules. Returns a list of valid relative paths and any walk error. Walk errors are wrapped with context (path + \"error walking\"). Relative-path computation failures are also wrapped. Directory entries matching ignore rules return `filepath.SkipDir`. Non-matching files append to the slice.\n\n**Walk sequence:**\n1. Iterate via `filepath.WalkDir` from `repoRoot`\n2. For each directory entry: skip if name starts with `.`, ends with `.egg-info`, or matches compiled GitIgnore patterns\n3. For each file entry: run both ignore-checks (compiled rules + dot-prefixed / `.egg-info` suffix), collect those that pass\n\n---\n\n## Git Integration (`git_tools.go`)\n\n### Command Execution — `RunGit`\n\n```go\nfunc RunGit(repoRoot string, args ...string) (string, error)\n```\n\nExecutes a git command in the specified repo directory with `--no-pager`. Returns trimmed stdout and any error.\n\n**Sequence:**\n1. Construct invocation: `git --no-pager \u003cargs\u003e` in `repoRoot`\n2. Spawn child process via `exec.Command(\"git\", ...)` — subprocess runs in `repoRoot`\n3. Capture stdout into `bytes.Buffer`, stderr separately buffered\n4. If `cmd.Run()` succeeds: trim whitespace from captured stdout, return as string result\n5. If `cmd.Run()` fails: trim whitespace from captured stderr, wrap with `fmt.Errorf(\"git command failed: %v, stderr: %s\", err, trimmedErr)`\n\nThe original error from `exec.Cmd` (e.g., `*exec.Error`, non-zero exit status) is embedded as `%v`; stderr text appended. No panic occurs on failure.\n\n### Repository Validation — `VerifyGitRepo`\n\n```go\nfunc VerifyGitRepo(repoRoot string) error\n```\n\nChecks that `git` is available and that `repoRoot` is inside a git working tree. Invokes `git rev-parse --is-inside-work-tree`. If the returned error is non-nil, wraps with: `fmt.Errorf(\"not a git repository (or any of the parent directories): %w\", err)`. Uses `%w` for wrapError-compatible wrapping (recoverable by unwrappers).\n\n---\n\n## Error Propagation Patterns\n\n### Disk I/O — File Operations\n\nAll functions interact exclusively with the local filesystem via standard OS calls (`os.ReadFile`, `os.CreateTemp`, `os.MkdirAll`, `os.Rename`, `os.Chmod`, `os.Open`). No network, database, or external API interactions exist.\n\n| Function | I/O Type | External System |\n|---|---|---|\n| `ReadFileSafely` | Disk read (`os.ReadFile`) | Local filesystem — target path resolved through `security.SafeResolve` before reading |\n| `WriteFileAtomic` | Disk write (create temp → sync → chmod → rename) | Local filesystem — writes to a `.tmp.*` intermediate file then renames into place |\n| `WriteFileSafely` | Disk read + disk write | Delegates path resolution and atomic write; no additional I/O beyond what the two sub-functions perform |\n| `EnsureGitignoreHasLockfile` | Disk read (`os.ReadFile`) + disk write (`WriteFileAtomic`) | Reads `.gitignore`, appends lockfile pattern, writes back atomically |\n| `LoadGitignore` | Disk read (`os.Open` + `bufio.Scanner`) | Reads `.gitignore`; returns `nil, nil` when file does not exist (no error) |\n| `DiscoverCodeFiles` | Disk walk (`filepath.WalkDir`) | Recursively walks the repository root; skips directories matching ignore rules or dot-prefixed names |\n\n### Process I/O — Git Operations\n\nThe only external interaction is subprocess execution against the installed `git` binary. Stdout captured to `bytes.Buffer`, stderr separately buffered. `cmd.Run()` executes the command and returns the exit error.\n\n| Function | System | Mechanism |\n|---|---|---|\n| `RunGit` | External process (git) | `exec.Command(\"git\", ...)` via `os/exec` — spawns child git subprocess in `repoRoot`. Stdout captured to `bytes.Buffer`, stderr separately buffered. `cmd.Run()` executes the command and returns the exit error. |\n| `VerifyGitRepo` | External process (git) | `RunGit(repoRoot, \"rev-parse\", \"--is-inside-work-tree\")` — delegates to `RunGit` for execution; wraps non-nil return value with repository classification error |\n\n---\n\n## Concurrency \u0026 State\n\nNo mutable state exists within any function scope. All operations are stateless and idempotent with respect to repository content. No goroutine spawning or concurrent access patterns observed in this module.\n\n---\n\n## Notable Observations\n\n- **TOCTOU mitigation**: `WriteFileAtomic` uses a temp file + rename pattern, which avoids race conditions on the target path during writes\n- **Sync before close**: `tmpFile.Sync()` is called before `Close()`, ensuring data reaches stable storage before the handle is released — relevant for crash safety but adds latency per write\n- **Deferred cleanup in WriteFileAtomic**: On any error, `tmpFile.Close()` and `os.Remove(tmpName)` run via defer. This prevents partial temp file leakage on failure paths. The flag `success` gates whether to trigger cleanup\n- **No panic/recover anywhere** — all errors are returned as typed values. No `panic`, no bare recovery blocks observed in this module or its tests" + ".": "# Code-Reducer: Technical Documentation\n\n## Module Responsibility\n\nCode-Reducer is a CLI-driven documentation agent that automatically generates and maintains project wikis. The system orchestrates an LLM-powered synthesis pipeline across four operational modes: **init** (first-time documentation build), **setup** (interactive configuration wizard), **update** (incremental wiki regeneration), and **update** (incremental wiki regeneration). All external behavior is delegated to the `cmd` package via `cmd.RootCmd.Execute()`.\n\n---\n\n## Data Flow\n\n```\nCLI Invocation\n │\n ▼\nRootCmd (root.go)\n │\n ├── ModeInit ──► initCmd ──► executeCommand(ModeInit) ──► engine.Run(ModeInit)\n │\n ├── ModeUpdate ──► updateCmd ──► executeCommand(ModeUpdate) ──► engine.Run(ModeUpdate)\n │\n └── ModeSetup ──► setupCmd ──► RunSetupFlow ──► config.SaveConfig\n```\n\n---\n\n## Subsystem: cmd\n\n### Command Registration\n\n#### init.go\n\nThe `init` command is registered at package scope via a global `*cobra.Command` variable. On execution, it delegates to `executeCommand` with `engine.ModeInit`. The engine mode triggers a repository scan and generates the first set of wiki markdown pages. No public API surface is exposed from this file; all identifiers are unexported.\n\n#### root.go\n\nThe root command (`RootCmd`) serves as the CLI application's top-level entry point. It is initialized once and never modified after creation. The root command binds two persistent flags (`modelIDFlag`, `numCtxFlag`) and provides the following unexported functions:\n\n- **`init()`** — Initializes the root command and registers all subcommands.\n- **`executeCommand(mode engine.RunMode)`** — Central dispatch function. Accepts a `engine.RunMode` and returns `error`. Aggregates errors from `os.Getwd()`, `tools.VerifyGitRepo`, `checkAndRunSetup`, `config.ResolveConfig`, `checkInitStatus`, and `runEngine`.\n- **`checkAndRunSetup(repoRoot string)`** — Returns `error` when the configuration file is missing and stdin is not a terminal.\n- **`checkInitStatus(repoRoot, docsDir string, mode engine.RunMode)`** — Returns `error` when the requested init/update mode conflicts with the project's initialization state.\n- **`runEngine(repoRoot string, cfg *config.Config, mode engine.RunMode)`** — Wraps engine errors with `fmt.Errorf(\"documentation run failed: %w\", err)`.\n\n#### setup.go\n\nThe `setup` command (`setupCmd`) implements an interactive CLI setup wizard. It generates or updates a `.code-reducer.yaml` configuration file by collecting the following values from the user:\n\n1. LLM Model ID\n2. Ollama Base URL\n3. Ollama Context Size (numeric validation with default fallback)\n4. Ignore patterns (comma-separated list; supports \"clear\"/\"none\" to reset)\n5. Documentation directory\n\nThe flow loads any existing configuration from the current working directory (falling back to an empty `config.Config{}` if the load fails), applies defaults, prompts sequentially, and persists the result via `config.SaveConfig`.\n\n#### update.go\n\nThe `update` command (`updateCmd`) performs incremental documentation updates. It identifies files modified since the last documented commit using cached SHA-256 hashes and regenerates the corresponding wiki pages. The operation is tied to the last documented commit as the baseline. No direct I/O is performed in this file; all logic is delegated to `internal/engine` via `executeCommand(engine.ModeUpdate)`.\n\n### Error Propagation\n\nAll functions in `cmd` return `error` type. Errors propagate up the call chain via return values. No panics or crashes are present. The cobra framework displays returned errors to the user and exits with a non-zero status code. Errors from `executeCommand` are returned directly to the cobra command framework via `RunE`.\n\n### State and Concurrency\n\nNo synchronization mechanisms are present. Global variables (`initCmd`, `setupCmd`, `updateCmd`) are declared at package scope and initialized once; they are never modified after initialization. No mutexes, locks, atomic types, or async patterns are used.\n\n---\n\n## Subsystem: internal\n\n### Configuration Subsystem (`internal/config`)\n\n#### Configuration Persistence (`io.go`)\n\n**Public API**\n\n| Function | Parameters | Return Types |\n|---|---|---|\n| `ConfigExists` | `cwd string` | `bool` |\n| `LoadConfig` | `cwd string` | `*Config`, `error` |\n| `SaveConfig` | `cwd string`, `cfg *Config` | `error` |\n\n**Internal Functions**\n\n| Function | Parameters | Return Types |\n|---|---|---|\n| `getConfigPath` | `cwd string` | `string` |\n| `formatYAML` | `data []byte` | `string` |\n\n**File System Interactions**\n\n- **Disk reads:** `os.Stat` in `ConfigExists`; `os.ReadFile` in `LoadConfig`.\n- **Disk writes:** `tools.WriteFileAtomic` in `SaveConfig`.\n- No network, database, or API interactions.\n\n**Error Propagation**\n\n- `LoadConfig` returns `(*Config, error)`. If `os.ReadFile` fails, the raw error is returned. If YAML parsing fails, the error is wrapped with `fmt.Errorf(\"failed to parse yaml config: %w\", err)`.\n- `SaveConfig` returns `error`. If `yaml.Marshal` fails, the error is wrapped with `fmt.Errorf(\"failed to marshal yaml: %w\", err)` before being passed to `tools.WriteFileAtomic`.\n- `ConfigExists` derives a boolean from `os.Stat` error check (`err == nil`).\n- `getConfigPath` and `formatYAML` have no error handling; failures result in panics.\n\n#### Configuration Resolution (`resolve.go`)\n\n**Public API**\n\n| Function | Parameters | Return Type |\n|---|---|---|\n| `ApplyDefaults` | `cfg *Config` | (none) |\n| `ResolveConfig` | `repoRoot string`, `modelIDFlag string`, `numCtxFlag string` | `(*Config, error)` |\n\n**Internal Functions**\n\n| Function | Parameters | Return Type |\n|---|---|---|\n| `deduplicate` | `a []string` | `[]string` |\n| `loadBaseConfig` | `repoRoot string` | `(*Config, error)` |\n| `resolveModelID` | `cfgVal string`, `flagVal string` | `string` |\n| `resolveBaseURL` | `cfgVal string` | `string` |\n| `parsePositiveInt` | `val string`, `errMsg string` | `(int, error)` |\n| `resolveNumCtx` | `cfgVal int`, `flagVal string` | `(int, error)` |\n\n**External I/O Interactions**\n\n- **Disk I/O:** `loadBaseConfig` calls `LoadConfig(repoRoot)` from `io.go`. If the file does not exist (`os.ErrNotExist`), an empty `Config{}` is returned instead of an error. Other read errors are wrapped and returned.\n- **Environment Variables:** Three environment variables are read via `os.Getenv`:\n - `CodeReducerModelIDEnvKey` in `resolveModelID`\n - `OllamaBaseURLEnvKey` in `resolveBaseURL`\n - `OllamaNumCtxEnvKey` in `resolveNumCtx`\n- No network, database, or API calls.\n\n**Error Propagation**\n\n- `ResolveConfig` returns errors from `loadBaseConfig` (which wraps non-`os.ErrNotExist` errors from `LoadConfig`) and from `resolveNumCtx` (which wraps errors from `parsePositiveInt`).\n- `ApplyDefaults` and `deduplicate` swallow all errors silently.\n- `resolveModelID` and `resolveBaseURL` return values without errors; callers handle the empty-string case implicitly.\n- `parsePositiveInt` returns a formatted error string via `fmt.Errorf`.\n- Errors are propagated via Go's `error` return convention; no panics or exceptions are used.\n\n#### Configuration Schema (`config.go`)\n\n**Constants**\n\n| Constant | Value |\n|---|---|\n| `ConfigFileName` | `.code-reducer.yaml` |\n| `configFilePerm` | `0600` |\n| `OllamaDefaultBaseURL` | `http://localhost:11434` |\n| `OllamaDefaultModelID` | `ornith:9b` |\n| `OllamaDefaultNumCtx` | `8192` |\n| `DefaultDocsDir` | `wiki` |\n| `CodeReducerModelIDEnvKey` | `CODE_REDUCER_MODEL_ID` |\n| `OllamaBaseURLEnvKey` | `OLLAMA_BASE_URL` |\n| `OllamaNumCtxEnvKey` | `OLLAMA_NUM_CTX` |\n\n**Default Prompts**\n\nThe following multi-line string constants define system prompts for downstream synthesis steps:\n\n- `DefaultSystemPrompt` — Code-Reducer role and rules (extract facts, no speculation, no code blocks)\n- `DefaultModuleSynthesisPrompt` — Folder summary generation\n- `DefaultArchitecturePrompt` — Global architecture document generation\n- `DefaultQuickstartPrompt` — Developer quickstart guide generation\n- `DefaultFileFactConsolidationPrompt` — Fact consolidation and deduplication\n\n**Structs**\n\n| Type | Fields |\n|---|---|\n| `ExtractionStep` | `Name` string, `Prompt` string |\n| `Config` | `ModelID` string, `OllamaBaseURL` string, `OllamaNumCtx` int, `DocsDir` string, `SystemPrompt` string, `ModuleSynthesisPrompt` string, `ArchitecturePrompt` string, `QuickstartPrompt` string, `FileFactConsolidationPrompt` string, `ExtractionSteps` []ExtractionStep, `Ignore` []string |\n\n**Default Extraction Steps**\n\n`DefaultExtractionSteps` is a `[]ExtractionStep` containing: `API_SIGNATURES`, `BUSINESS_LOGIC`, `STATE_AND_CONCURRENCY`, `ERRORS_AND_SIDE_EFFECTS`.\n\n#### Resolution Priority Ordering\n\nThe following priority ordering is enforced by `ResolveConfig` and its helpers:\n\n| Field | Priority Order |\n|---|---|\n| `ModelID` | CLI flag → `CODE_REDUCER_MODEL_ID` env var → config file value |\n| `OllamaBaseURL` | `OLLAMA_BASE_URL` env var → config file value |\n| `OllamaNumCtx` | CLI flag → `OLLAMA_NUM_CTX` env var → config file value |\n| `DocsDir` | System default (`wiki`) when empty/zero |\n| `ExtractionSteps` | System default when empty |\n| All prompt fields | System default when empty |\n\n`DocsDir` is additionally trimmed and path-cleaned during default application.\n\n---\n\n## Engine Subsystem (`internal/engine`)\n\n### Core Infrastructure\n\n#### LLM Client (`client.go`)\n\nThe `llmCaller` interface abstracts LLM invocation. The unexported `llmClient` type implements it by serializing requests to the Ollama `/api/chat` endpoint and returning text responses.\n\n```go\ntype llmCaller interface {\n CallLLM(ctx context.Context, systemPrompt, userPrompt string) (string, error)\n NumCtx() int\n}\n```\n\n`CallLLM` sends an HTTP POST with a 10-minute timeout. On a 200 OK, the full response body is read; on non-200, up to 1024 bytes of the error body are captured. Marshaling, request creation, and response parsing errors are wrapped with `fmt.Errorf`. The `onStatus` callback fires when `done_reason == \"length\"`.\n\n#### Metadata Cache (`cache.go`)\n\n`MetadataCache` persists file state and configuration across engine runs. It stores a version, a `stepsHash` derived from model ID and all prompt templates, per-file SHA-256 hashes with associated facts, and module-to-path mappings.\n\n```go\ntype FileCacheEntry struct {\n SHA256 string\n Facts string\n}\n\ntype MetadataCache struct {\n Version int\n StepsHash string\n Files map[string]FileCacheEntry\n Modules map[string]string\n}\n```\n\n`loadMetadataCache` reads `.metadata.json` from `docsDir`. If absent, it returns an empty cache; if the version is incompatible, it resets. `saveMetadataCache` writes indented JSON. `IsInitialized` checks file existence. `computeSHA256` hashes arbitrary file paths.\n\n### Tree Construction \u0026 Change Detection\n\n#### Directory Tree (`tree.go`)\n\n`DirNode` represents a directory containing files and child directories. `buildTree` parses a file slice into a hierarchical `*DirNode` structure.\n\n```go\ntype DirNode struct {\n Path string\n Files []string\n Children map[string]*DirNode\n}\n```\n\n`FileChange` tracks a file's status (`StatusAdded`, `StatusModified`, `StatusDeleted`) and path. `propagateAffected` walks the tree upward, marking ancestors affected when a descendant is affected. `isNodeAffected` checks whether a node's files appear in the changed set or its module is missing from cache. `determineAffected` combines changed files with cache state to produce a final affected-dirs map.\n\n#### Orchestrator (`orchestrator.go`)\n\nThe `orchestrator` struct holds the `llmCaller` and coordinates the full pipeline. `pipelineState` is created fresh per invocation and contains `repoRoot`, `cfg`, `cache`, `affectedDirs`, `precalculatedHashes`, and `onStatus`.\n\n```go\ntype orchestrator struct {\n client llmCaller\n}\n\ntype pipelineState struct {\n client llmCaller\n repoRoot string\n cfg *config.Config\n cache *MetadataCache\n affectedDirs map[string]bool\n precalculatedHashes map[string]string\n onStatus StatusFunc\n}\n```\n\n`Run` is the entry point. It acquires a lock via `security.AcquireLock`, loads `.gitignore`, loads the metadata cache, and dispatches to `RunInit` or `RunUpdate`. `RunInit` builds the tree, hashes all code files, synthesizes the root summary, generates `architecture.md` and `quickstart.md`, updates `AGENTS.md`, and persists the cache. `RunUpdate` detects file changes, prunes stale cache entries, and regenerates only affected modules.\n\n`setupPipeline` initializes the cache and affected-dirs map. `teardownPipeline` cleans up stale module files and empty directories. `hashCodeFiles` computes SHA-256 hashes for a file list. `collectTreeDirs` returns the set of directories present in the tree. `updateAgentGuidelines` appends documentation references to `AGENTS.md`.\n\n### Hierarchical Synthesis\n\n#### Synthesis Pipeline (`synthesize.go`)\n\n`extractFileFacts` invokes the LLM for each extraction step per chunk of a file. It strips outer markdown fences, filters \"none\" facts, and caches results keyed by file SHA-256. Read failures are swallowed via `onStatus`; LLM and chunking failures propagate.\n\n`synthesizeChildren` processes child directories recursively, synthesizing subsystem summaries before individual files. `collectComponents` combines subsystem summaries and file facts into a unified component list. `synthesizeNode` reduces all components for a directory into a final summary, writes it to `modules/\u003cpath\u003e.md`, and persists the cache. If no components are produced, the cached summary is deleted and the markdown file is removed.\n\n```go\nfunc extractFileFacts(ctx context.Context, p *pipelineState, f string, siblingFiles []string) (string, error)\nfunc synthesizeChildren(ctx context.Context, p *pipelineState, node *DirNode) ([]string, error)\nfunc collectComponents(ctx context.Context, p *pipelineState, node *DirNode, subsystems []string) ([]string, error)\nfunc synthesizeNode(ctx context.Context, p *pipelineState, node *DirNode) (string, error)\n```\n\n`buildExtractionPrompt` constructs the user prompt for a single extraction step. `maxExtractionOverhead` computes the maximum characters available for extraction given system prompt and file context. `toSafeMarkdownFilename` sanitizes a module path for use as a filename.\n\n#### Text Reduction \u0026 Chunking (`chunking.go`)\n\n`reduceWithLLM` calls the LLM to reduce a batch of items. `reduceInChunks` splits oversized items with an 800-character overlap, batches remaining items, and reduces each batch. If total output reaches 95% of input, reduction stops and results are concatenated. Otherwise, the intermediate results are recursively reduced.\n\n`chunkTextWithOverlap` splits text with overlap validation. `stripOuterMarkdownFence` removes surrounding code fences. `calculateFileLimit` computes the maximum processing characters from context window minus system prompt overhead. `isNoneFact` filters empty, \"none\", and \"n/a\" facts. `expandOversizedItems` splits items exceeding the limit. `batchItems` groups remaining items. `countRunes` counts Unicode code points. `reduceItems` orchestrates the reduction with a custom function.\n\n### Error Handling Summary\n\n| Operation | Failure Mode | Handling |\n|-----------|-------------|----------|\n| Lock acquisition | `security.AcquireLock` fails | Fatal error returned |\n| Gitignore load | `tools.LoadGitignore` fails | Fatal error returned |\n| Code discovery | `tools.DiscoverCodeFiles` fails | Fatal error returned |\n| File read (extraction) | `tools.ReadFileSafely` fails | Swallowed, logged via `onStatus` |\n| LLM call | `CallLLM` fails | Propagated as wrapped error |\n| Chunking | Invalid `maxRunes` or `overlapRunes` | Returned as error |\n| Disk write | `tools.WriteFileSafely` fails | Fatal error returned |\n| Stale cleanup | `os.Remove` fails | Silently ignored |\n| Cache save | Marshaling or write fails | Fatal error returned |\n| HTTP non-200 | Response body read | Wrapped with status code |\n| Context cancellation | Any operation | Propagated as error |\n\n---\n\n## Security Subsystem (`security.go`)\n\n### Module Responsibility\n\nThis module provides two non-negotiable security primitives for a repository-level tool:\n\n1. **Path traversal prevention** — `SafeResolve` ensures any input path resolves strictly inside the repository root, even when symlinks are involved.\n2. **Process-level locking** — `AcquireLock` / `SimpleLock` / `Unlock` prevents concurrent execution by multiple processes of the same tool.\n\n**Data flow:** `SafeResolve(repoRoot, inputPath)` → `(string, error)`. `AcquireLock(repoRoot)` → `(*SimpleLock, error)`. `(*SimpleLock).Unlock()` → `error`. No global mutable state exists; all constants and sentinel errors are immutable.\n\n### Path Traversal Prevention\n\n#### `SafeResolve(repoRoot, inputPath string) (string, error)`\n\nCleans `inputPath` and verifies the resolved result lies strictly inside the repository root.\n\n**Algorithm:**\n\n1. `filepath.Abs(repoRoot)` — canonicalize the repository root to an absolute path.\n2. `filepath.EvalSymlinks(repoRoot)` — resolve symlinks on the repository root itself.\n3. Walk up the input path from the root, stopping at the first physically existing ancestor (checked via `os.Lstat`).\n4. `filepath.EvalSymlinks` on that existing ancestor.\n5. Rebuild the full path by appending the non-existent suffix components.\n6. `filepath.Rel` computes the relative path from the resolved root to the rebuilt path. If the result is not `\"\"`, return `ErrPathTraversal`.\n\n**Error propagation:** Returns `(string, error)`. Errors are wrapped with `fmt.Errorf` using `%w` for chainability. `ErrPathTraversal` is returned directly as a sentinel.\n\n### Process-Level Locking\n\n#### `AcquireLock(repoRoot string) (*SimpleLock, error)`\n\nAcquires a simple file lock in `repoRoot` using `O_EXCL` for atomicity.\n\n**Algorithm:**\n\n1. Resolve the lock file path using `SafeResolve`.\n2. `os.OpenFile` with `O_WRONLY|O_CREATE|O_EXCL`. If the file already exists, return `ErrLockHeld`.\n3. Write the current process PID (`os.Getpid`) to the file.\n4. Return a `*SimpleLock` handle.\n\n**Error propagation:** If `OpenFile` fails with `os.IsExist`, wraps `ErrLockHeld`. If `Write` fails, the file is closed and removed before returning a wrapped error.\n\n#### `(*SimpleLock) Unlock() error`\n\nReleases the lock by closing the file and removing it. Idempotent and thread-safe.\n\n**Algorithm:**\n\n1. Acquire the internal `sync.Mutex` (`l.mu`).\n2. If `l.closed` is true, return `nil`.\n3. Close the file descriptor.\n4. Remove the lock file. If both close and remove fail, return the error from the non-nil one.\n\n**Error propagation:** If close succeeds but remove fails (and is not `os.IsNotExist`), the remove error is assigned to `err`. If close fails, the remove error is ignored.\n\n### Types\n\n#### `SimpleLock`\n\n```go\ntype SimpleLock struct {\n lockPath string\n file *os.File\n mu sync.Mutex\n closed bool\n}\n```\n\n- `lockPath` — absolute path to the lock file.\n- `file` — open file descriptor for the lock.\n- `mu` — protects `closed` field.\n- `closed` — set to `true` in `Unlock()`; guards idempotency.\n\n### Constants and Errors\n\n**Constants**\n\n- `LockFileName` = `\".code-reducer.lock\"`\n- `defaultFilePerm` = `0644`\n\n**Error Variables**\n\n- `ErrPathTraversal` = `errors.New(\"security violation: path traversal detected\")`\n- `ErrLockHeld` = `errors.New(\"lock is already held by another process\")`\n\n### External I/O Interactions\n\nAll operations target the local filesystem. No network, database, or API interactions.\n\n| Operation | Purpose |\n|-----------|---------|\n| `filepath.Abs` | Resolves absolute path for `repoRoot` |\n| `filepath.EvalSymlinks` | Resolves symlinks on `repoRoot` and intermediate ancestors |\n| `os.Lstat` | Checks existence of path components during traversal resolution |\n| `os.OpenFile` | Opens the lock file with `O_WRONLY|O_CREATE|O_EXCL` |\n| `os.Remove` | Removes the lock file during `Unlock()` and on write failure in `AcquireLock()` |\n| `os.Getpid` | Reads process ID (no external I/O) |\n| `filepath.Rel` | Computes relative path to verify containment |\n\n### Synchronization\n\n- `SimpleLock.Unlock()` uses `sync.Mutex` (`l.mu`) to protect the `closed` field.\n- `AcquireLock()` is a package-level function that creates a new `SimpleLock` instance; it does not use any synchronization mechanism.\n\n### Error Handling Summary\n\n- Errors are not logged or printed; they propagate to the caller.\n- No `panic`/`recover` is used.\n- Sentinel errors (`ErrPathTraversal`, `ErrLockHeld`) are returned directly.\n- All other errors are wrapped with `fmt.Errorf` using `%w` for chainability.\n\n---\n\n## Tools Subsystem (`internal/tools`)\n\n### Module Responsibility\n\nThis module provides safe filesystem operations and repository verification utilities for source code management workflows. It operates on a repository root (`repoRoot`) and resolves virtual paths through a security layer to prevent path traversal attacks. All operations are disk I/O only—no network, database, or external API calls.\n\n**Data Flow:**\n\n```\ncaller → virtualPath → security.SafeResolve → filesystem path → os.* operations\n```\n\nAll public functions accept a `repoRoot` parameter and resolve virtual paths before accessing the filesystem. Errors are propagated as return values; no panics occur.\n\n### File Tools (`file_tools.go`)\n\n#### Safe File Access\n\n**`FileExistsSafely(repoRoot string, virtualPath string) bool`**\n\nChecks whether a file exists at the resolved filesystem path. Swallows `os.Stat` errors and returns `false` on any failure.\n\n**`ReadFileSafely(repoRoot string, virtualPath string) ([]byte, error)`**\n\nReads file contents from the resolved filesystem path. Returns the file data and an error. Swallows `os.Open` errors.\n\n**`RemoveFileSafely(repoRoot string, virtualPath string) error`**\n\nRemoves a file at the resolved filesystem path. Ignores `os.IsNotExist` errors (file may already be absent). Propagates all other errors.\n\n#### Atomic File Writes\n\n**`WriteFileAtomic(targetPath string, data []byte, perm os.FileMode) error`**\n\nWrites data to `targetPath` atomically: writes to a temporary file first, then renames to the target path. Wraps all sub-errors with `fmt.Errorf(\"failed to %s: %w\", ...)`.\n\n**`WriteFileSafely(repoRoot string, virtualPath string, content []byte) error`**\n\nWrites content to the resolved filesystem path. Uses atomic write semantics.\n\n#### Gitignore Integration\n\n**`EnsureGitignoreHasLockfile(repoRoot string) error`**\n\nEnsures `.code-reducer.lock` is included in `.gitignore` to prevent it from being committed.\n\n**`LoadGitignore(repoRoot string) ([]string, error)`**\n\nLoads `.gitignore` patterns from the repository root. Returns the parsed patterns and an error.\n\n**`ShouldIgnoreFile(relPath string, gitIgnore *ignore.GitIgnore) bool`**\n\nDetermines whether a relative path matches any loaded gitignore patterns. Uses `ignore.CompileIgnoreLines` for pattern parsing (in-memory, no external I/O).\n\n### Global State\n\n| Variable | Type | Mutability |\n|---|---|---|\n| `skippedFileNames` | `map[string]bool` | Immutable after init |\n| `skippedExtensions` | `map[string]bool` | Immutable after init |\n| `skippedTestSuffixes` | `[]string` | Immutable after init |\n\nThese maps are initialized once and never modified. They define the multi-layer file filtering rules applied during file discovery.\n\n### Git Tools (`git_tools.go`)\n\n#### Repository Verification\n\n**`VerifyGitRepo(repoRoot string) error`**\n\nVerifies whether `repoRoot` resides inside a git repository by executing `git rev-parse --is-inside-work-tree` in the specified directory.\n\n**Process Execution Details:**\n\n- Spawns `git` as a child process via `exec.Command(\"git\", \"--no-pager\", \"rev-parse\", \"--is-inside-work-tree\")`\n- Redirects child process stderr to a `bytes.Buffer` for capture\n- Sets child process working directory to `repoRoot`\n\n**Error Handling:**\n\n- If `cmd.Run()` returns an error, the function wraps it with `fmt.Errorf` and returns the wrapped error (including captured stderr content)\n- If `cmd.Run()` returns nil, returns nil (success)\n- No panics or swallowed errors", + "cmd": "# cmd Package Architecture\n\n## Module Responsibility\n\nThe `cmd` package implements the CLI entry point for a documentation agent that writes and maintains project wikis. It orchestrates the documentation engine across three operational modes: **init** (first-time documentation build), **setup** (interactive configuration wizard), and **update** (incremental wiki regeneration). All commands are registered as `*cobra.Command` instances and share a common execution path through `executeCommand`, which dispatches to the engine based on the requested `engine.RunMode`.\n\n## Data Flow\n\n```\nCLI Invocation\n │\n ▼\nRootCmd (root.go)\n │\n ├── ModeInit ──► initCmd ──► executeCommand(ModeInit) ──► engine.Run(ModeInit)\n │\n ├── ModeUpdate ──► updateCmd ──► executeCommand(ModeUpdate) ──► engine.Run(ModeUpdate)\n │\n └── ModeSetup ──► setupCmd ──► RunSetupFlow ──► config.SaveConfig\n```\n\n## Command Registration\n\n### init.go\n\nThe `init` command is registered at package scope via a global `*cobra.Command` variable. On execution, it delegates to `executeCommand` with `engine.ModeInit`. The engine mode triggers a repository scan and generates the first set of wiki markdown pages. No public API surface is exposed from this file; all identifiers are unexported.\n\n### root.go\n\nThe root command (`RootCmd`) serves as the CLI application's top-level entry point. It is initialized once and never modified after creation. The root command binds two persistent flags (`modelIDFlag`, `numCtxFlag`) and provides the following unexported functions:\n\n- **`init()`** — Initializes the root command and registers all subcommands.\n- **`executeCommand(mode engine.RunMode)`** — Central dispatch function. Accepts a `engine.RunMode` and returns `error`. Aggregates errors from `os.Getwd()`, `tools.VerifyGitRepo`, `checkAndRunSetup`, `config.ResolveConfig`, `checkInitStatus`, and `runEngine`.\n- **`checkAndRunSetup(repoRoot string)`** — Returns `error` when the configuration file is missing and stdin is not a terminal.\n- **`checkInitStatus(repoRoot, docsDir string, mode engine.RunMode)`** — Returns `error` when the requested init/update mode conflicts with the project's initialization state.\n- **`runEngine(repoRoot string, cfg *config.Config, mode engine.RunMode)`** — Wraps engine errors with `fmt.Errorf(\"documentation run failed: %w\", err)`.\n\n### setup.go\n\nThe `setup` command (`setupCmd`) implements an interactive CLI setup wizard. It generates or updates a `.code-reducer.yaml` configuration file by collecting the following values from the user:\n\n1. LLM Model ID\n2. Ollama Base URL\n3. Ollama Context Size (numeric validation with default fallback)\n4. Ignore patterns (comma-separated list; supports \"clear\"/\"none\" to reset)\n5. Documentation directory\n\nThe flow loads any existing configuration from the current working directory (falling back to an empty `config.Config{}` if the load fails), applies defaults, prompts sequentially, and persists the result via `config.SaveConfig`.\n\n### update.go\n\nThe `update` command (`updateCmd`) performs incremental documentation updates. It identifies files modified since the last documented commit using cached SHA-256 hashes and regenerates the corresponding wiki pages. The operation is tied to the last documented commit as the baseline. No direct I/O is performed in this file; all logic is delegated to `internal/engine` via `executeCommand(engine.ModeUpdate)`.\n\n## Error Propagation\n\nAll functions in `cmd` return `error` type. Errors propagate up the call chain via return values. No panics or crashes are present. The cobra framework displays returned errors to the user and exits with a non-zero status code. Errors from `executeCommand` are returned directly to the cobra command framework via `RunE`.\n\n## State and Concurrency\n\nNo synchronization mechanisms are present. Global variables (`initCmd`, `setupCmd`, `updateCmd`) are declared at package scope and initialized once; they are never modified after initialization. No mutexes, locks, atomic types, or async patterns are used.", + "internal": "# internal — Codebase Documentation Synthesis Engine\n\n## Module Responsibility\n\nThe `internal` package implements an LLM-driven pipeline that automatically generates and maintains codebase documentation. It coordinates four subsystems:\n\n- **config** — Configuration lifecycle: persistence, resolution, and default application.\n- **engine** — Orchestration of the synthesis pipeline: cache management, change detection, hierarchical synthesis, and final persistence.\n- **security** — Path traversal prevention and process-level locking.\n- **tools** — Safe filesystem operations and repository verification utilities.\n\n**Data flow:** `ResolveConfig` (config) → `Run` (engine) → `security.SafeResolve` / `security.AcquireLock` / `tools.*` for all filesystem access. The engine reads configuration, acquires a lock, parses `.gitignore`, discovers code files, computes SHA-256 hashes, builds a directory tree, and dispatches to `RunInit` or `RunUpdate`. The orchestrator coordinates cache loading, change detection, hierarchical synthesis, and final persistence.\n\n---\n\n## Configuration Subsystem (`internal/config`)\n\n### Configuration Persistence (`io.go`)\n\n**Public API**\n\n| Function | Parameters | Return Types |\n|---|---|---|\n| `ConfigExists` | `cwd string` | `bool` |\n| `LoadConfig` | `cwd string` | `*Config`, `error` |\n| `SaveConfig` | `cwd string`, `cfg *Config` | `error` |\n\n**Internal Functions**\n\n| Function | Parameters | Return Types |\n|---|---|---|\n| `getConfigPath` | `cwd string` | `string` |\n| `formatYAML` | `data []byte` | `string` |\n\n**File System Interactions**\n\n- **Disk reads:** `os.Stat` in `ConfigExists`; `os.ReadFile` in `LoadConfig`.\n- **Disk writes:** `tools.WriteFileAtomic` in `SaveConfig`.\n- No network, database, or API interactions.\n\n**Error Propagation**\n\n- `LoadConfig` returns `(*Config, error)`. If `os.ReadFile` fails, the raw error is returned. If YAML parsing fails, the error is wrapped with `fmt.Errorf(\"failed to parse yaml config: %w\", err)`.\n- `SaveConfig` returns `error`. If `yaml.Marshal` fails, the error is wrapped with `fmt.Errorf(\"failed to marshal yaml: %w\", err)` before being passed to `tools.WriteFileAtomic`.\n- `ConfigExists` derives a boolean from `os.Stat` error check (`err == nil`).\n- `getConfigPath` and `formatYAML` have no error handling; failures result in panics.\n\n### Configuration Resolution (`resolve.go`)\n\n**Public API**\n\n| Function | Parameters | Return Type |\n|---|---|---|\n| `ApplyDefaults` | `cfg *Config` | (none) |\n| `ResolveConfig` | `repoRoot string`, `modelIDFlag string`, `numCtxFlag string` | `(*Config, error)` |\n\n**Internal Functions**\n\n| Function | Parameters | Return Type |\n|---|---|---|\n| `deduplicate` | `a []string` | `[]string` |\n| `loadBaseConfig` | `repoRoot string` | `(*Config, error)` |\n| `resolveModelID` | `cfgVal string`, `flagVal string` | `string` |\n| `resolveBaseURL` | `cfgVal string` | `string` |\n| `parsePositiveInt` | `val string`, `errMsg string` | `(int, error)` |\n| `resolveNumCtx` | `cfgVal int`, `flagVal string` | `(int, error)` |\n\n**External I/O Interactions**\n\n- **Disk I/O:** `loadBaseConfig` calls `LoadConfig(repoRoot)` from `io.go`. If the file does not exist (`os.ErrNotExist`), an empty `Config{}` is returned instead of an error. Other read errors are wrapped and returned.\n- **Environment Variables:** Three environment variables are read via `os.Getenv`:\n - `CodeReducerModelIDEnvKey` in `resolveModelID`\n - `OllamaBaseURLEnvKey` in `resolveBaseURL`\n - `OllamaNumCtxEnvKey` in `resolveNumCtx`\n- No network, database, or API calls.\n\n**Error Propagation**\n\n- `ResolveConfig` returns errors from `loadBaseConfig` (which wraps non-`os.ErrNotExist` errors from `LoadConfig`) and from `resolveNumCtx` (which wraps errors from `parsePositiveInt`).\n- `ApplyDefaults` and `deduplicate` swallow all errors silently.\n- `resolveModelID` and `resolveBaseURL` return values without errors; callers handle the empty-string case implicitly.\n- `parsePositiveInt` returns a formatted error string via `fmt.Errorf`.\n- Errors are propagated via Go's `error` return convention; no panics or exceptions are used.\n\n### Configuration Schema (`config.go`)\n\n**Constants**\n\n| Constant | Value |\n|---|---|\n| `ConfigFileName` | `.code-reducer.yaml` |\n| `configFilePerm` | `0600` |\n| `OllamaDefaultBaseURL` | `http://localhost:11434` |\n| `OllamaDefaultModelID` | `ornith:9b` |\n| `OllamaDefaultNumCtx` | `8192` |\n| `DefaultDocsDir` | `wiki` |\n| `CodeReducerModelIDEnvKey` | `CODE_REDUCER_MODEL_ID` |\n| `OllamaBaseURLEnvKey` | `OLLAMA_BASE_URL` |\n| `OllamaNumCtxEnvKey` | `OLLAMA_NUM_CTX` |\n\n**Default Prompts**\n\nThe following multi-line string constants define system prompts for downstream synthesis steps:\n\n- `DefaultSystemPrompt` — Code-Reducer role and rules (extract facts, no speculation, no code blocks)\n- `DefaultModuleSynthesisPrompt` — Folder summary generation\n- `DefaultArchitecturePrompt` — Global architecture document generation\n- `DefaultQuickstartPrompt` — Developer quickstart guide generation\n- `DefaultFileFactConsolidationPrompt` — Fact consolidation and deduplication\n\n**Structs**\n\n| Type | Fields |\n|---|---|\n| `ExtractionStep` | `Name` string, `Prompt` string |\n| `Config` | `ModelID` string, `OllamaBaseURL` string, `OllamaNumCtx` int, `DocsDir` string, `SystemPrompt` string, `ModuleSynthesisPrompt` string, `ArchitecturePrompt` string, `QuickstartPrompt` string, `FileFactConsolidationPrompt` string, `ExtractionSteps` []ExtractionStep, `Ignore` []string |\n\n**Default Extraction Steps**\n\n`DefaultExtractionSteps` is a `[]ExtractionStep` containing: `API_SIGNATURES`, `BUSINESS_LOGIC`, `STATE_AND_CONCURRENCY`, `ERRORS_AND_SIDE_EFFECTS`.\n\n### Resolution Priority Ordering\n\nThe following priority ordering is enforced by `ResolveConfig` and its helpers:\n\n| Field | Priority Order |\n|---|---|\n| `ModelID` | CLI flag → `CODE_REDUCER_MODEL_ID` env var → config file value |\n| `OllamaBaseURL` | `OLLAMA_BASE_URL` env var → config file value |\n| `OllamaNumCtx` | CLI flag → `OLLAMA_NUM_CTX` env var → config file value |\n| `DocsDir` | System default (`wiki`) when empty/zero |\n| `ExtractionSteps` | System default when empty |\n| All prompt fields | System default when empty |\n\n`DocsDir` is additionally trimmed and path-cleaned during default application.\n\n---\n\n## Engine Subsystem (`internal/engine`)\n\n### Core Infrastructure\n\n#### LLM Client (`client.go`)\n\nThe `llmCaller` interface abstracts LLM invocation. The unexported `llmClient` type implements it by serializing requests to the Ollama `/api/chat` endpoint and returning text responses.\n\n```go\ntype llmCaller interface {\n CallLLM(ctx context.Context, systemPrompt, userPrompt string) (string, error)\n NumCtx() int\n}\n```\n\n`CallLLM` sends an HTTP POST with a 10-minute timeout. On a 200 OK, the full response body is read; on non-200, up to 1024 bytes of the error body are captured. Marshaling, request creation, and response parsing errors are wrapped with `fmt.Errorf`. The `onStatus` callback fires when `done_reason == \"length\"`.\n\n#### Metadata Cache (`cache.go`)\n\n`MetadataCache` persists file state and configuration across engine runs. It stores a version, a `stepsHash` derived from model ID and all prompt templates, per-file SHA-256 hashes with associated facts, and module-to-path mappings.\n\n```go\ntype FileCacheEntry struct {\n SHA256 string\n Facts string\n}\n\ntype MetadataCache struct {\n Version int\n StepsHash string\n Files map[string]FileCacheEntry\n Modules map[string]string\n}\n```\n\n`loadMetadataCache` reads `.metadata.json` from `docsDir`. If absent, it returns an empty cache; if the version is incompatible, it resets. `saveMetadataCache` writes indented JSON. `IsInitialized` checks file existence. `computeSHA256` hashes arbitrary file paths.\n\n### Tree Construction \u0026 Change Detection\n\n#### Directory Tree (`tree.go`)\n\n`DirNode` represents a directory containing files and child directories. `buildTree` parses a file slice into a hierarchical `*DirNode` structure.\n\n```go\ntype DirNode struct {\n Path string\n Files []string\n Children map[string]*DirNode\n}\n```\n\n`FileChange` tracks a file's status (`StatusAdded`, `StatusModified`, `StatusDeleted`) and path. `propagateAffected` walks the tree upward, marking ancestors affected when a descendant is affected. `isNodeAffected` checks whether a node's files appear in the changed set or its module is missing from cache. `determineAffected` combines changed files with cache state to produce a final affected-dirs map.\n\n#### Orchestrator (`orchestrator.go`)\n\nThe `orchestrator` struct holds the `llmCaller` and coordinates the full pipeline. `pipelineState` is created fresh per invocation and contains `repoRoot`, `cfg`, `cache`, `affectedDirs`, `precalculatedHashes`, and `onStatus`.\n\n```go\ntype orchestrator struct {\n client llmCaller\n}\n\ntype pipelineState struct {\n client llmCaller\n repoRoot string\n cfg *config.Config\n cache *MetadataCache\n affectedDirs map[string]bool\n precalculatedHashes map[string]string\n onStatus StatusFunc\n}\n```\n\n`Run` is the entry point. It acquires a lock via `security.AcquireLock`, loads `.gitignore`, loads the metadata cache, and dispatches to `RunInit` or `RunUpdate`. `RunInit` builds the tree, hashes all code files, synthesizes the root summary, generates `architecture.md` and `quickstart.md`, updates `AGENTS.md`, and persists the cache. `RunUpdate` detects file changes, prunes stale cache entries, and regenerates only affected modules.\n\n`setupPipeline` initializes the cache and affected-dirs map. `teardownPipeline` cleans up stale module files and empty directories. `hashCodeFiles` computes SHA-256 hashes for a file list. `collectTreeDirs` returns the set of directories present in the tree. `updateAgentGuidelines` appends documentation references to `AGENTS.md`.\n\n### Hierarchical Synthesis\n\n#### Synthesis Pipeline (`synthesize.go`)\n\n`extractFileFacts` invokes the LLM for each extraction step per chunk of a file. It strips outer markdown fences, filters \"none\" facts, and caches results keyed by file SHA-256. Read failures are swallowed via `onStatus`; LLM and chunking failures propagate.\n\n`synthesizeChildren` processes child directories recursively, synthesizing subsystem summaries before individual files. `collectComponents` combines subsystem summaries and file facts into a unified component list. `synthesizeNode` reduces all components for a directory into a final summary, writes it to `modules/\u003cpath\u003e.md`, and persists the cache. If no components are produced, the cached summary is deleted and the markdown file is removed.\n\n```go\nfunc extractFileFacts(ctx context.Context, p *pipelineState, f string, siblingFiles []string) (string, error)\nfunc synthesizeChildren(ctx context.Context, p *pipelineState, node *DirNode) ([]string, error)\nfunc collectComponents(ctx context.Context, p *pipelineState, node *DirNode, subsystems []string) ([]string, error)\nfunc synthesizeNode(ctx context.Context, p *pipelineState, node *DirNode) (string, error)\n```\n\n`buildExtractionPrompt` constructs the user prompt for a single extraction step. `maxExtractionOverhead` computes the maximum characters available for extraction given system prompt and file context. `toSafeMarkdownFilename` sanitizes a module path for use as a filename.\n\n#### Text Reduction \u0026 Chunking (`chunking.go`)\n\n`reduceWithLLM` calls the LLM to reduce a batch of items. `reduceInChunks` splits oversized items with an 800-character overlap, batches remaining items, and reduces each batch. If total output reaches 95% of input, reduction stops and results are concatenated. Otherwise, the intermediate results are recursively reduced.\n\n`chunkTextWithOverlap` splits text with overlap validation. `stripOuterMarkdownFence` removes surrounding code fences. `calculateFileLimit` computes the maximum processing characters from context window minus system prompt overhead. `isNoneFact` filters empty, \"none\", and \"n/a\" facts. `expandOversizedItems` splits items exceeding the limit. `batchItems` groups remaining items. `countRunes` counts Unicode code points. `reduceItems` orchestrates the reduction with a custom function.\n\n### Error Handling Summary\n\n| Operation | Failure Mode | Handling |\n|-----------|-------------|----------|\n| Lock acquisition | `security.AcquireLock` fails | Fatal error returned |\n| Gitignore load | `tools.LoadGitignore` fails | Fatal error returned |\n| Code discovery | `tools.DiscoverCodeFiles` fails | Fatal error returned |\n| File read (extraction) | `tools.ReadFileSafely` fails | Swallowed, logged via `onStatus` |\n| LLM call | `CallLLM` fails | Propagated as wrapped error |\n| Chunking | Invalid `maxRunes` or `overlapRunes` | Returned as error |\n| Disk write | `tools.WriteFileSafely` fails | Fatal error returned |\n| Stale cleanup | `os.Remove` fails | Silently ignored |\n| Cache save | Marshaling or write fails | Fatal error returned |\n| HTTP non-200 | Response body read | Wrapped with status code |\n| Context cancellation | Any operation | Propagated as error |\n\n---\n\n## Security Subsystem (`security.go`)\n\n### Module Responsibility\n\nThis module provides two non-negotiable security primitives for a repository-level tool:\n\n1. **Path traversal prevention** — `SafeResolve` ensures any input path resolves strictly inside the repository root, even when symlinks are involved.\n2. **Process-level locking** — `AcquireLock` / `SimpleLock` / `Unlock` prevents concurrent execution by multiple processes of the same tool.\n\n**Data flow:** `SafeResolve(repoRoot, inputPath)` → `(string, error)`. `AcquireLock(repoRoot)` → `(*SimpleLock, error)`. `(*SimpleLock).Unlock()` → `error`. No global mutable state exists; all constants and sentinel errors are immutable.\n\n### Path Traversal Prevention\n\n#### `SafeResolve(repoRoot, inputPath string) (string, error)`\n\nCleans `inputPath` and verifies the resolved result lies strictly inside the repository root.\n\n**Algorithm:**\n\n1. `filepath.Abs(repoRoot)` — canonicalize the repository root to an absolute path.\n2. `filepath.EvalSymlinks(repoRoot)` — resolve symlinks on the repository root itself.\n3. Walk up the input path from the root, stopping at the first physically existing ancestor (checked via `os.Lstat`).\n4. `filepath.EvalSymlinks` on that existing ancestor.\n5. Rebuild the full path by appending the non-existent suffix components.\n6. `filepath.Rel` computes the relative path from the resolved root to the rebuilt path. If the result is not `\"\"`, return `ErrPathTraversal`.\n\n**Error propagation:** Returns `(string, error)`. Errors are wrapped with `fmt.Errorf` using `%w` for chainability. `ErrPathTraversal` is returned directly as a sentinel.\n\n### Process-Level Locking\n\n#### `AcquireLock(repoRoot string) (*SimpleLock, error)`\n\nAcquires a simple file lock in `repoRoot` using `O_EXCL` for atomicity.\n\n**Algorithm:**\n\n1. Resolve the lock file path using `SafeResolve`.\n2. `os.OpenFile` with `O_WRONLY|O_CREATE|O_EXCL`. If the file already exists, return `ErrLockHeld`.\n3. Write the current process PID (`os.Getpid`) to the file.\n4. Return a `*SimpleLock` handle.\n\n**Error propagation:** If `OpenFile` fails with `os.IsExist`, wraps `ErrLockHeld`. If `Write` fails, the file is closed and removed before returning a wrapped error.\n\n#### `(*SimpleLock) Unlock() error`\n\nReleases the lock by closing the file and removing it. Idempotent and thread-safe.\n\n**Algorithm:**\n\n1. Acquire the internal `sync.Mutex` (`l.mu`).\n2. If `l.closed` is true, return `nil`.\n3. Close the file descriptor.\n4. Remove the lock file. If both close and remove fail, return the error from the non-nil one.\n\n**Error propagation:** If close succeeds but remove fails (and is not `os.IsNotExist`), the remove error is assigned to `err`. If close fails, the remove error is ignored.\n\n### Types\n\n#### `SimpleLock`\n\n```go\ntype SimpleLock struct {\n lockPath string\n file *os.File\n mu sync.Mutex\n closed bool\n}\n```\n\n- `lockPath` — absolute path to the lock file.\n- `file` — open file descriptor for the lock.\n- `mu` — protects `closed` field.\n- `closed` — set to `true` in `Unlock()`; guards idempotency.\n\n### Constants and Errors\n\n**Constants**\n\n- `LockFileName` = `\".code-reducer.lock\"`\n- `defaultFilePerm` = `0644`\n\n**Error Variables**\n\n- `ErrPathTraversal` = `errors.New(\"security violation: path traversal detected\")`\n- `ErrLockHeld` = `errors.New(\"lock is already held by another process\")`\n\n### External I/O Interactions\n\nAll operations target the local filesystem. No network, database, or API interactions.\n\n| Operation | Purpose |\n|-----------|---------|\n| `filepath.Abs` | Resolves absolute path for `repoRoot` |\n| `filepath.EvalSymlinks` | Resolves symlinks on `repoRoot` and intermediate ancestors |\n| `os.Lstat` | Checks existence of path components during traversal resolution |\n| `os.OpenFile` | Opens the lock file with `O_WRONLY|O_CREATE|O_EXCL` |\n| `os.Remove` | Removes the lock file during `Unlock()` and on write failure in `AcquireLock()` |\n| `os.Getpid` | Reads process ID (no external I/O) |\n| `filepath.Rel` | Computes relative path to verify containment |\n\n### Synchronization\n\n- `SimpleLock.Unlock()` uses `sync.Mutex` (`l.mu`) to protect the `closed` field.\n- `AcquireLock()` is a package-level function that creates a new `SimpleLock` instance; it does not use any synchronization mechanism.\n\n### Error Handling Summary\n\n- Errors are not logged or printed; they propagate to the caller.\n- No `panic`/`recover` is used.\n- Sentinel errors (`ErrPathTraversal`, `ErrLockHeld`) are returned directly.\n- All other errors are wrapped with `fmt.Errorf` using `%w` for chainability.\n\n---\n\n## Tools Subsystem (`internal/tools`)\n\n### Module Responsibility\n\nThis module provides safe filesystem operations and repository verification utilities for source code management workflows. It operates on a repository root (`repoRoot`) and resolves virtual paths through a security layer to prevent path traversal attacks. All operations are disk I/O only—no network, database, or external API calls.\n\n**Data Flow:**\n\n```\ncaller → virtualPath → security.SafeResolve → filesystem path → os.* operations\n```\n\nAll public functions accept a `repoRoot` parameter and resolve virtual paths before accessing the filesystem. Errors are propagated as return values; no panics occur.\n\n### File Tools (`file_tools.go`)\n\n#### Safe File Access\n\n**`FileExistsSafely(repoRoot string, virtualPath string) bool`**\n\nChecks whether a file exists at the resolved filesystem path. Swallows `os.Stat` errors and returns `false` on any failure.\n\n**`ReadFileSafely(repoRoot string, virtualPath string) ([]byte, error)`**\n\nReads file contents from the resolved filesystem path. Returns the file data and an error. Swallows `os.Open` errors.\n\n**`RemoveFileSafely(repoRoot string, virtualPath string) error`**\n\nRemoves a file at the resolved filesystem path. Ignores `os.IsNotExist` errors (file may already be absent). Propagates all other errors.\n\n#### Atomic File Writes\n\n**`WriteFileAtomic(targetPath string, data []byte, perm os.FileMode) error`**\n\nWrites data to `targetPath` atomically: writes to a temporary file first, then renames to the target path. Wraps all sub-errors with `fmt.Errorf(\"failed to %s: %w\", ...)`.\n\n**`WriteFileSafely(repoRoot string, virtualPath string, content []byte) error`**\n\nWrites content to the resolved filesystem path. Uses atomic write semantics.\n\n#### Gitignore Integration\n\n**`EnsureGitignoreHasLockfile(repoRoot string) error`**\n\nEnsures `.code-reducer.lock` is included in `.gitignore` to prevent it from being committed.\n\n**`LoadGitignore(repoRoot string) ([]string, error)`**\n\nLoads `.gitignore` patterns from the repository root. Returns the parsed patterns and an error.\n\n**`ShouldIgnoreFile(relPath string, gitIgnore *ignore.GitIgnore) bool`**\n\nDetermines whether a relative path matches any loaded gitignore patterns. Uses `ignore.CompileIgnoreLines` for pattern parsing (in-memory, no external I/O).\n\n### Global State\n\n| Variable | Type | Mutability |\n|---|---|---|\n| `skippedFileNames` | `map[string]bool` | Immutable after init |\n| `skippedExtensions` | `map[string]bool` | Immutable after init |\n| `skippedTestSuffixes` | `[]string` | Immutable after init |\n\nThese maps are initialized once and never modified. They define the multi-layer file filtering rules applied during file discovery.\n\n### Git Tools (`git_tools.go`)\n\n#### Repository Verification\n\n**`VerifyGitRepo(repoRoot string) error`**\n\nVerifies whether `repoRoot` resides inside a git repository by executing `git rev-parse --is-inside-work-tree` in the specified directory.\n\n**Process Execution Details:**\n\n- Spawns `git` as a child process via `exec.Command(\"git\", \"--no-pager\", \"rev-parse\", \"--is-inside-work-tree\")`\n- Redirects child process stderr to a `bytes.Buffer` for capture\n- Sets child process working directory to `repoRoot`\n\n**Error Handling:**\n\n- If `cmd.Run()` returns an error, the function wraps it with `fmt.Errorf` and returns the wrapped error (including captured stderr content)\n- If `cmd.Run()` returns nil, returns nil (success)\n- No panics or swallowed errors", + "internal/config": "# internal/config\n\n## Module Responsibility\n\nThe `internal/config` package manages the configuration lifecycle for Code-Reducer, an AI-driven code analysis and documentation generation tool. It handles three concerns: (1) persistence of a `.code-reducer.yaml` configuration file, (2) resolution of configuration values from multiple sources with defined priority ordering, and (3) application of system defaults to empty or zero-valued fields.\n\n**Data flow:** `ResolveConfig` is the primary entry point. It loads a base configuration from disk via `loadBaseConfig` (which delegates to `LoadConfig` in `io.go`), applies system defaults via `ApplyDefaults`, then resolves individual fields (`ModelID`, `OllamaBaseURL`, `OllamaNumCtx`) by merging CLI flags, environment variables, and config file values. The `Ignore` slice is deduplicated via `deduplicate`. The resolved `Config` struct is returned to callers.\n\n---\n\n## Configuration Persistence (`io.go`)\n\n### Public API\n\n| Function | Parameters | Return Types |\n|---|---|---|\n| `ConfigExists` | `cwd string` | `bool` |\n| `LoadConfig` | `cwd string` | `*Config`, `error` |\n| `SaveConfig` | `cwd string`, `cfg *Config` | `error` |\n\n### Internal Functions\n\n| Function | Parameters | Return Types |\n|---|---|---|\n| `getConfigPath` | `cwd string` | `string` |\n| `formatYAML` | `data []byte` | `string` |\n\n### File System Interactions\n\n- **Disk reads:** `os.Stat` in `ConfigExists`; `os.ReadFile` in `LoadConfig`.\n- **Disk writes:** `tools.WriteFileAtomic` in `SaveConfig`.\n- No network, database, or API interactions.\n\n### Error Propagation\n\n- `LoadConfig` returns `(*Config, error)`. If `os.ReadFile` fails, the raw error is returned. If YAML parsing fails, the error is wrapped with `fmt.Errorf(\"failed to parse yaml config: %w\", err)`.\n- `SaveConfig` returns `error`. If `yaml.Marshal` fails, the error is wrapped with `fmt.Errorf(\"failed to marshal yaml: %w\", err)` before being passed to `tools.WriteFileAtomic`.\n- `ConfigExists` derives a boolean from `os.Stat` error check (`err == nil`).\n- `getConfigPath` and `formatYAML` have no error handling; failures result in panics.\n\n---\n\n## Configuration Resolution (`resolve.go`)\n\n### Public API\n\n| Function | Parameters | Return Type |\n|---|---|---|\n| `ApplyDefaults` | `cfg *Config` | (none) |\n| `ResolveConfig` | `repoRoot string`, `modelIDFlag string`, `numCtxFlag string` | `(*Config, error)` |\n\n### Internal Functions\n\n| Function | Parameters | Return Type |\n|---|---|---|\n| `deduplicate` | `a []string` | `[]string` |\n| `loadBaseConfig` | `repoRoot string` | `(*Config, error)` |\n| `resolveModelID` | `cfgVal string`, `flagVal string` | `string` |\n| `resolveBaseURL` | `cfgVal string` | `string` |\n| `parsePositiveInt` | `val string`, `errMsg string` | `(int, error)` |\n| `resolveNumCtx` | `cfgVal int`, `flagVal string` | `(int, error)` |\n\n### External I/O Interactions\n\n- **Disk I/O:** `loadBaseConfig` calls `LoadConfig(repoRoot)` from `io.go`. If the file does not exist (`os.ErrNotExist`), an empty `Config{}` is returned instead of an error. Other read errors are wrapped and returned.\n- **Environment Variables:** Three environment variables are read via `os.Getenv`:\n - `CodeReducerModelIDEnvKey` in `resolveModelID`\n - `OllamaBaseURLEnvKey` in `resolveBaseURL`\n - `OllamaNumCtxEnvKey` in `resolveNumCtx`\n- No network, database, or API calls.\n\n### Error Propagation\n\n- `ResolveConfig` returns errors from `loadBaseConfig` (which wraps non-`os.ErrNotExist` errors from `LoadConfig`) and from `resolveNumCtx` (which wraps errors from `parsePositiveInt`).\n- `ApplyDefaults` and `deduplicate` swallow all errors silently.\n- `resolveModelID` and `resolveBaseURL` return values without errors; callers handle the empty-string case implicitly.\n- `parsePositiveInt` returns a formatted error string via `fmt.Errorf`.\n- Errors are propagated via Go's `error` return convention; no panics or exceptions are used.\n\n---\n\n## Configuration Schema (`config.go`)\n\n### Constants\n\n| Constant | Value |\n|---|---|\n| `ConfigFileName` | `.code-reducer.yaml` |\n| `ConfigFileName` | `ConfigFileName` (referenced in `io.go` and `resolve.go`) |\n| `configFilePerm` | `0600` |\n| `configFilePerm` | `configFilePerm` (referenced in `io.go`) |\n| `OllamaDefaultBaseURL` | `http://localhost:11434` |\n| `OllamaDefaultModelID` | `ornith:9b` |\n| `OllamaDefaultNumCtx` | `8192` |\n| `DefaultDocsDir` | `wiki` |\n| `CodeReducerModelIDEnvKey` | `CODE_REDUCER_MODEL_ID` |\n| `OllamaBaseURLEnvKey` | `OLLAMA_BASE_URL` |\n| `OllamaNumCtxEnvKey` | `OLLAMA_NUM_CTX` |\n\n### Default Prompts\n\nThe following multi-line string constants define system prompts for downstream synthesis steps:\n\n- `DefaultSystemPrompt` — Code-Reducer role and rules (extract facts, no speculation, no code blocks)\n- `DefaultModuleSynthesisPrompt` — Folder summary generation\n- `DefaultArchitecturePrompt` — Global architecture document generation\n- `DefaultQuickstartPrompt` — Developer quickstart guide generation\n- `DefaultFileFactConsolidationPrompt` — Fact consolidation and deduplication\n\n### Structs\n\n| Type | Fields |\n|---|---|\n| `ExtractionStep` | `Name` string, `Prompt` string |\n| `Config` | `ModelID` string, `OllamaBaseURL` string, `OllamaNumCtx` int, `DocsDir` string, `SystemPrompt` string, `ModuleSynthesisPrompt` string, `ArchitecturePrompt` string, `QuickstartPrompt` string, `FileFactConsolidationPrompt` string, `ExtractionSteps` []ExtractionStep, `Ignore` []string |\n\n### Default Extraction Steps\n\n`DefaultExtractionSteps` is a `[]ExtractionStep` containing: `API_SIGNATURES`, `BUSINESS_LOGIC`, `STATE_AND_CONCURRENCY`, `ERRORS_AND_SIDE_EFFECTS`.\n\n---\n\n## Resolution Priority Ordering\n\nThe following priority ordering is enforced by `ResolveConfig` and its helpers:\n\n| Field | Priority Order |\n|---|---|\n| `ModelID` | CLI flag → `CODE_REDUCER_MODEL_ID` env var → config file value |\n| `OllamaBaseURL` | `OLLAMA_BASE_URL` env var → config file value |\n| `OllamaNumCtx` | CLI flag → `OLLAMA_NUM_CTX` env var → config file value |\n| `DocsDir` | System default (`wiki`) when empty/zero |\n| `ExtractionSteps` | System default when empty |\n| All prompt fields | System default when empty |\n\n`DocsDir` is additionally trimmed and path-cleaned during default application.", + "internal/engine": "# internal/engine — Codebase Documentation Synthesis Engine\n\n## Module Responsibility\n\nThe `internal/engine` package implements an LLM-driven pipeline that automatically generates and maintains codebase documentation (architecture overview, quickstart guide, module details, and agent guidelines). The engine operates in two modes:\n\n- **Init** — Performs a full synthesis of the entire codebase from scratch.\n- **Update** — Detects file changes since the last run and regenerates only the affected documentation.\n\nData flows from the repository root through a lock acquisition, gitignore parsing, code discovery, SHA-256 hashing, and tree construction. The orchestrator coordinates cache loading, change detection, hierarchical synthesis, and final persistence.\n\n---\n\n## Core Infrastructure\n\n### LLM Client (`client.go`)\n\nThe `llmCaller` interface abstracts LLM invocation. The unexported `llmClient` type implements it by serializing requests to the Ollama `/api/chat` endpoint and returning text responses.\n\n```go\ntype llmCaller interface {\n CallLLM(ctx context.Context, systemPrompt, userPrompt string) (string, error)\n NumCtx() int\n}\n```\n\n`CallLLM` sends an HTTP POST with a 10-minute timeout. On a 200 OK, the full response body is read; on non-200, up to 1024 bytes of the error body are captured. Marshaling, request creation, and response parsing errors are wrapped with `fmt.Errorf`. The `onStatus` callback fires when `done_reason == \"length\"`.\n\n### Metadata Cache (`cache.go`)\n\n`MetadataCache` persists file state and configuration across engine runs. It stores a version, a `stepsHash` derived from model ID and all prompt templates, per-file SHA-256 hashes with associated facts, and module-to-path mappings.\n\n```go\ntype FileCacheEntry struct {\n SHA256 string\n Facts string\n}\n\ntype MetadataCache struct {\n Version int\n StepsHash string\n Files map[string]FileCacheEntry\n Modules map[string]string\n}\n```\n\n`loadMetadataCache` reads `.metadata.json` from `docsDir`. If absent, it returns an empty cache; if the version is incompatible, it resets. `saveMetadataCache` writes indented JSON. `IsInitialized` checks file existence. `computeSHA256` hashes arbitrary file paths.\n\n---\n\n## Tree Construction \u0026 Change Detection\n\n### Directory Tree (`tree.go`)\n\n`DirNode` represents a directory containing files and child directories. `buildTree` parses a file slice into a hierarchical `*DirNode` structure.\n\n```go\ntype DirNode struct {\n Path string\n Files []string\n Children map[string]*DirNode\n}\n```\n\n`FileChange` tracks a file's status (`StatusAdded`, `StatusModified`, `StatusDeleted`) and path. `propagateAffected` walks the tree upward, marking ancestors affected when a descendant is affected. `isNodeAffected` checks whether a node's files appear in the changed set or its module is missing from cache. `determineAffected` combines changed files with cache state to produce a final affected-dirs map.\n\n### Orchestrator (`orchestrator.go`)\n\nThe `orchestrator` struct holds the `llmCaller` and coordinates the full pipeline. `pipelineState` is created fresh per invocation and contains `repoRoot`, `cfg`, `cache`, `affectedDirs`, `precalculatedHashes`, and `onStatus`.\n\n```go\ntype orchestrator struct {\n client llmCaller\n}\n\ntype pipelineState struct {\n client llmCaller\n repoRoot string\n cfg *config.Config\n cache *MetadataCache\n affectedDirs map[string]bool\n precalculatedHashes map[string]string\n onStatus StatusFunc\n}\n```\n\n`Run` is the entry point. It acquires a lock via `security.AcquireLock`, loads `.gitignore`, loads the metadata cache, and dispatches to `RunInit` or `RunUpdate`. `RunInit` builds the tree, hashes all code files, synthesizes the root summary, generates `architecture.md` and `quickstart.md`, updates `AGENTS.md`, and persists the cache. `RunUpdate` detects file changes, prunes stale cache entries, and regenerates only affected modules.\n\n`setupPipeline` initializes the cache and affected-dirs map. `teardownPipeline` cleans up stale module files and empty directories. `hashCodeFiles` computes SHA-256 hashes for a file list. `collectTreeDirs` returns the set of directories present in the tree. `updateAgentGuidelines` appends documentation references to `AGENTS.md`.\n\n---\n\n## Hierarchical Synthesis\n\n### Synthesis Pipeline (`synthesize.go`)\n\n`extractFileFacts` invokes the LLM for each extraction step per chunk of a file. It strips outer markdown fences, filters \"none\" facts, and caches results keyed by file SHA-256. Read failures are swallowed via `onStatus`; LLM and chunking failures propagate.\n\n`synthesizeChildren` processes child directories recursively, synthesizing subsystem summaries before individual files. `collectComponents` combines subsystem summaries and file facts into a unified component list. `synthesizeNode` reduces all components for a directory into a final summary, writes it to `modules/\u003cpath\u003e.md`, and persists the cache. If no components are produced, the cached summary is deleted and the markdown file is removed.\n\n```go\nfunc extractFileFacts(ctx context.Context, p *pipelineState, f string, siblingFiles []string) (string, error)\nfunc synthesizeChildren(ctx context.Context, p *pipelineState, node *DirNode) ([]string, error)\nfunc collectComponents(ctx context.Context, p *pipelineState, node *DirNode, subsystems []string) ([]string, error)\nfunc synthesizeNode(ctx context.Context, p *pipelineState, node *DirNode) (string, error)\n```\n\n`buildExtractionPrompt` constructs the user prompt for a single extraction step. `maxExtractionOverhead` computes the maximum characters available for extraction given system prompt and file context. `toSafeMarkdownFilename` sanitizes a module path for use as a filename.\n\n### Text Reduction \u0026 Chunking (`chunking.go`)\n\n`reduceWithLLM` calls the LLM to reduce a batch of items. `reduceInChunks` splits oversized items with an 800-character overlap, batches remaining items, and reduces each batch. If total output reaches 95% of input, reduction stops and results are concatenated. Otherwise, the intermediate results are recursively reduced.\n\n`chunkTextWithOverlap` splits text with overlap validation. `stripOuterMarkdownFence` removes surrounding code fences. `calculateFileLimit` computes the maximum processing characters from context window minus system prompt overhead. `isNoneFact` filters empty, \"none\", and \"n/a\" facts. `expandOversizedItems` splits items exceeding the limit. `batchItems` groups remaining items. `countRunes` counts Unicode code points. `reduceItems` orchestrates the reduction with a custom function.\n\n---\n\n## Error Handling Summary\n\n| Operation | Failure Mode | Handling |\n|-----------|-------------|----------|\n| Lock acquisition | `security.AcquireLock` fails | Fatal error returned |\n| Gitignore load | `tools.LoadGitignore` fails | Fatal error returned |\n| Code discovery | `tools.DiscoverCodeFiles` fails | Fatal error returned |\n| File read (extraction) | `tools.ReadFileSafely` fails | Swallowed, logged via `onStatus` |\n| LLM call | `CallLLM` fails | Propagated as wrapped error |\n| Chunking | Invalid `maxRunes` or `overlapRunes` | Returned as error |\n| Disk write | `tools.WriteFileSafely` fails | Fatal error returned |\n| Stale cleanup | `os.Remove` fails | Silently ignored |\n| Cache save | Marshaling or write fails | Fatal error returned |\n| HTTP non-200 | Response body read | Wrapped with status code |\n| Context cancellation | Any operation | Propagated as error |", + "internal/security": "# security.go — Repository Security Module\n\n## Module Responsibility\n\nThis module provides two non-negotiable security primitives for a repository-level tool:\n\n1. **Path traversal prevention** — `SafeResolve` ensures any input path resolves strictly inside the repository root, even when symlinks are involved.\n2. **Process-level locking** — `AcquireLock` / `SimpleLock` / `Unlock` prevents concurrent execution by multiple processes of the same tool.\n\n**Data flow:** `SafeResolve(repoRoot, inputPath)` → `(string, error)`. `AcquireLock(repoRoot)` → `(*SimpleLock, error)`. `(*SimpleLock).Unlock()` → `error`. No global mutable state exists; all constants and sentinel errors are immutable.\n\n---\n\n## Path Traversal Prevention\n\n### `SafeResolve(repoRoot, inputPath string) (string, error)`\n\nCleans `inputPath` and verifies the resolved result lies strictly inside the repository root.\n\n**Algorithm:**\n\n1. `filepath.Abs(repoRoot)` — canonicalize the repository root to an absolute path.\n2. `filepath.EvalSymlinks(repoRoot)` — resolve symlinks on the repository root itself.\n3. Walk up the input path from the root, stopping at the first physically existing ancestor (checked via `os.Lstat`).\n4. `filepath.EvalSymlinks` on that existing ancestor.\n5. Rebuild the full path by appending the non-existent suffix components.\n6. `filepath.Rel` computes the relative path from the resolved root to the rebuilt path. If the result is not `\"\"`, return `ErrPathTraversal`.\n\n**Error propagation:** Returns `(string, error)`. Errors are wrapped with `fmt.Errorf` using `%w` for chainability. `ErrPathTraversal` is returned directly as a sentinel.\n\n---\n\n## Process-Level Locking\n\n### `AcquireLock(repoRoot string) (*SimpleLock, error)`\n\nAcquires a simple file lock in `repoRoot` using `O_EXCL` for atomicity.\n\n**Algorithm:**\n\n1. Resolve the lock file path using `SafeResolve`.\n2. `os.OpenFile` with `O_WRONLY|O_CREATE|O_EXCL`. If the file already exists, return `ErrLockHeld`.\n3. Write the current process PID (`os.Getpid`) to the file.\n4. Return a `*SimpleLock` handle.\n\n**Error propagation:** If `OpenFile` fails with `os.IsExist`, wraps `ErrLockHeld`. If `Write` fails, the file is closed and removed before returning a wrapped error.\n\n### `(*SimpleLock) Unlock() error`\n\nReleases the lock by closing the file and removing it. Idempotent and thread-safe.\n\n**Algorithm:**\n\n1. Acquire the internal `sync.Mutex` (`l.mu`).\n2. If `l.closed` is true, return `nil`.\n3. Close the file descriptor.\n4. Remove the lock file. If both close and remove fail, return the error from the non-nil one.\n\n**Error propagation:** If close succeeds but remove fails (and is not `os.IsNotExist`), the remove error is assigned to `err`. If close fails, the remove error is ignored.\n\n---\n\n## Types\n\n### `SimpleLock`\n\n```go\ntype SimpleLock struct {\n lockPath string\n file *os.File\n mu sync.Mutex\n closed bool\n}\n```\n\n- `lockPath` — absolute path to the lock file.\n- `file` — open file descriptor for the lock.\n- `mu` — protects `closed` field.\n- `closed` — set to `true` in `Unlock()`; guards idempotency.\n\n---\n\n## Constants and Errors\n\n### Constants\n\n- `LockFileName` = `\".code-reducer.lock\"`\n- `defaultFilePerm` = `0644`\n\n### Error Variables\n\n- `ErrPathTraversal` = `errors.New(\"security violation: path traversal detected\")`\n- `ErrLockHeld` = `errors.New(\"lock is already held by another process\")`\n\n---\n\n## External I/O Interactions\n\nAll operations target the local filesystem. No network, database, or API interactions.\n\n| Operation | Purpose |\n|-----------|---------|\n| `filepath.Abs` | Resolves absolute path for `repoRoot` |\n| `filepath.EvalSymlinks` | Resolves symlinks on `repoRoot` and intermediate ancestors |\n| `os.Lstat` | Checks existence of path components during traversal resolution |\n| `os.OpenFile` | Opens the lock file with `O_WRONLY|O_CREATE|O_EXCL` |\n| `os.Remove` | Removes the lock file during `Unlock()` and on write failure in `AcquireLock()` |\n| `os.Getpid` | Reads process ID (no external I/O) |\n| `filepath.Rel` | Computes relative path to verify containment |\n\n---\n\n## Synchronization\n\n- `SimpleLock.Unlock()` uses `sync.Mutex` (`l.mu`) to protect the `closed` field.\n- `AcquireLock()` is a package-level function that creates a new `SimpleLock` instance; it does not use any synchronization mechanism.\n\n---\n\n## Error Handling Summary\n\n- Errors are not logged or printed; they propagate to the caller.\n- No `panic`/`recover` is used.\n- Sentinel errors (`ErrPathTraversal`, `ErrLockHeld`) are returned directly.\n- All other errors are wrapped with `fmt.Errorf` using `%w` for chainability.", + "internal/tools": "# Internal Tools Module\n\n## Module Responsibility\n\nThis module provides safe filesystem operations and repository verification utilities for source code management workflows. It operates on a repository root (`repoRoot`) and resolves virtual paths through a security layer to prevent path traversal attacks. All operations are disk I/O only—no network, database, or external API calls.\n\n**Data Flow:**\n\n```\ncaller → virtualPath → security.SafeResolve → filesystem path → os.* operations\n```\n\nAll public functions accept a `repoRoot` parameter and resolve virtual paths before accessing the filesystem. Errors are propagated as return values; no panics occur.\n\n---\n\n## File Tools (`file_tools.go`)\n\n### Safe File Access\n\n**`FileExistsSafely(repoRoot string, virtualPath string) bool`**\n\nChecks whether a file exists at the resolved filesystem path. Swallows `os.Stat` errors and returns `false` on any failure.\n\n**`ReadFileSafely(repoRoot string, virtualPath string) ([]byte, error)`**\n\nReads file contents from the resolved filesystem path. Returns the file data and an error. Swallows `os.Open` errors.\n\n**`RemoveFileSafely(repoRoot string, virtualPath string) error`**\n\nRemoves a file at the resolved filesystem path. Ignores `os.IsNotExist` errors (file may already be absent). Propagates all other errors.\n\n### Atomic File Writes\n\n**`WriteFileAtomic(targetPath string, data []byte, perm os.FileMode) error`**\n\nWrites data to `targetPath` atomically: writes to a temporary file first, then renames to the target path. Wraps all sub-errors with `fmt.Errorf(\"failed to %s: %w\", ...)`.\n\n**`WriteFileSafely(repoRoot string, virtualPath string, content []byte) error`**\n\nWrites content to the resolved filesystem path. Uses atomic write semantics.\n\n### Gitignore Integration\n\n**`EnsureGitignoreHasLockfile(repoRoot string) error`**\n\nEnsures `.code-reducer.lock` is included in `.gitignore` to prevent it from being committed.\n\n**`LoadGitignore(repoRoot string) ([]string, error)`**\n\nLoads `.gitignore` patterns from the repository root. Returns the parsed patterns and an error.\n\n**`ShouldIgnoreFile(relPath string, gitIgnore *ignore.GitIgnore) bool`**\n\nDetermines whether a relative path matches any loaded gitignore patterns. Uses `ignore.CompileIgnoreLines` for pattern parsing (in-memory, no external I/O).\n\n### Global State\n\n| Variable | Type | Mutability |\n|---|---|---|\n| `skippedFileNames` | `map[string]bool` | Immutable after init |\n| `skippedExtensions` | `map[string]bool` | Immutable after init |\n| `skippedTestSuffixes` | `[]string` | Immutable after init |\n\nThese maps are initialized once and never modified. They define the multi-layer file filtering rules applied during file discovery.\n\n---\n\n## Git Tools (`git_tools.go`)\n\n### Repository Verification\n\n**`VerifyGitRepo(repoRoot string) error`**\n\nVerifies whether `repoRoot` resides inside a git repository by executing `git rev-parse --is-inside-work-tree` in the specified directory.\n\n**Process Execution Details:**\n\n- Spawns `git` as a child process via `exec.Command(\"git\", \"--no-pager\", \"rev-parse\", \"--is-inside-work-tree\")`\n- Redirects child process stderr to a `bytes.Buffer` for capture\n- Sets child process working directory to `repoRoot`\n\n**Error Handling:**\n\n- If `cmd.Run()` returns an error, the function wraps it with `fmt.Errorf` and returns the wrapped error (including captured stderr content)\n- If `cmd.Run()` returns nil, returns nil (success)\n- No panics or swallowed errors" } } \ No newline at end of file diff --git a/wiki/architecture.md b/wiki/architecture.md index 69037fe..c5a185c 100644 --- a/wiki/architecture.md +++ b/wiki/architecture.md @@ -1,50 +1,75 @@ # Architecture Overview -Code-Reducer employs a hierarchical Map-Reduce architecture to transform source code into structured documentation. The system transitions from granular file analysis to high-level architectural synthesis through a recursive reduction process. - ## System Boundaries -* **Input/Interface**: The system accepts local source code within a git repository, user input via `os.Stdin`, configuration via `.code-reducer.yaml`, environment variables, and CLI flags. -* **External Dependencies**: - * **LLM Provider**: Communicates with local LLM instances via the `api/chat` protocol (Ollama). - * **Frontend Assets**: `mkdocs.yml` facilitates documentation deployment by fetching assets from `cdn.jsdelivr.net` and `cdnjs.cloudflare.com`. -* **Output**: Produces structured Markdown files in a designated `docsDir` and a `MetadataCache` in JSON format. - -## Module Interaction and Data Flow - -### 1. Orchestration Layer (`cmd`) -The `cmd` module serves as the system entry point, managing the execution lifecycle: -* **Validation**: Verifies the working directory is a git repository. -* **State Enforcement**: Uses `engine.IsInitialized` to switch between `ModeInit` (full scan) and `ModeUpdate` (incremental analysis). -* **Dispatch**: Resolves the configuration and hands off execution to the `engine`. - -### 2. Configuration Management (`internal/config`) -The `config` module handles the hierarchical resolution of settings: -* **Precedence**: CLI Flags $\rightarrow$ Environment Variables $\rightarrow$ Configuration File $\rightarrow$ Defaults. -* **Schema**: Manages parameters for LLM interaction (e.g., `OllamaNumCtx`) and prompt templates used during the `ExtractionStep` phases. -* **Persistence**: Ensures atomic writes to disk using a temporary file and rename pattern. - -### 3. Map-Reduce Engine (`internal/engine`) -The core engine executes the hierarchical synthesis via a `DirNode` tree structure: - -**Change Detection & Propagation:** -1. **Leaf Level**: The `Orchestrator` compares current file SHA256 hashes against the `MetadataCache`. -2. **Propagation**: If a change is detected, the "affected" status propagates up the `DirNode` tree from the file level to the directory and module levels. - -**Hierarchical Synthesis (The Map-Reduce Cycle):** -* **Map (Extraction)**: - * Files are processed through the `Synthesizer`. - * **Chunking**: If content exceeds the context window (governed by a `0.75 contextWindowAllocRatio`), the system triggers recursive reduction. - * **Recursion Termination**: The `Chunking` process terminates when the output size is $\ge$ 95% of the input size. -* **Reduce (Aggregation)**: - * Extracted information is aggregated from individual files $\rightarrow$ Modules $\rightarrow$ Directories $\rightarrow$ High-level Architecture. - -## Data Summary - -| Component | Responsibility | Primary Data/Format | -| :--- | :--- | :--- | -| `cmd` | Lifecycle & Dispatch | CLI Args, `config.Config` | -| `config` | Parameter Resolution | YAML, Environment Variables | -| `engine` | Hierarchical Synthesis | `DirNode` Tree, `MetadataCache` (JSON) | -| `engine/LLM` | Content Reduction | HTTP `api/chat` | -| `Output` | Documentation | Markdown | \ No newline at end of file +Code-Reducer is a CLI-driven documentation agent that automatically generates and maintains project wikis. It operates on a git repository root and produces markdown documentation in a configurable output directory (default: `wiki/`). The system has no network, database, or external API dependencies beyond an LLM backend (Ollama by default). + +The system boundary is the repository root. All filesystem operations resolve virtual paths through a security layer that prevents path traversal, even when symlinks are involved. The tool runs as a single process with no async patterns, no global mutable state, and no synchronization mechanisms. + +## Module Interaction Map + +``` +cmd/ ──► orchestrates CLI entry point, dispatches to engine +internal/config/ ──► configuration persistence and resolution +internal/engine/ ──► synthesis pipeline orchestration +internal/security/ ──► path traversal prevention, process locking +internal/tools/ ──► safe filesystem operations, git verification +``` + +All modules communicate via function parameters and return values. No shared mutable state exists between packages. The `cmd` package is the only entry point; all other packages are internal. + +## Operational Modes + +Three modes are exposed via the CLI: + +- **init** — first-time documentation build. Scans the repository, hashes all code files, synthesizes summaries hierarchically, generates `architecture.md` and `quickstart.md`, updates `AGENTS.md`, and persists the metadata cache. +- **setup** — interactive configuration wizard. Collects LLM model ID, Ollama base URL, context size, ignore patterns, and docs directory from the user. Persists via `config.SaveConfig`. +- **update** — incremental regeneration. Identifies files modified since the last documented commit using cached SHA-256 hashes and regenerates only affected modules. + +## Configuration Resolution + +Configuration is resolved in priority order: CLI flag → environment variable → config file → system default. The config file is `.code-reducer.yaml` with `0600` permissions. Three environment variables are supported: `CODE_REDUCER_MODEL_ID`, `OLLAMA_BASE_URL`, `OLLAMA_NUM_CTX`. + +The config schema includes LLM settings, extraction step definitions, prompt templates, ignore patterns, and the docs directory. Extraction steps are a fixed list: `API_SIGNATURES`, `BUSINESS_LOGIC`, `STATE_AND_CONCURRENCY`, `ERRORS_AND_SIDE_EFFECTS`. + +## Security Primitives + +Two non-negotiable security primitives protect the system: + +1. **Path traversal prevention** — `SafeResolve(repoRoot, inputPath)` canonicalizes the repository root, resolves symlinks on the root and intermediate ancestors, walks up the input path to the first physically existing ancestor, rebuilds the full path, and verifies the result lies strictly inside the repository root via `filepath.Rel`. Returns `ErrPathTraversal` if containment fails. + +2. **Process-level locking** — `AcquireLock(repoRoot)` opens a lock file with `O_WRONLY|O_CREATE|O_EXCL` for atomicity, writes the process PID, and returns a `*SimpleLock` handle. `Unlock()` closes the file descriptor and removes the lock file. Uses `sync.Mutex` to protect the closed state. + +## Synthesis Pipeline + +The engine orchestrates a hierarchical synthesis pipeline: + +1. **Tree construction** — `buildTree` parses a file slice into a hierarchical `*DirNode` structure. +2. **Change detection** — SHA-256 hashes are computed for all code files. The metadata cache stores per-file hashes with associated facts. On update, files modified since the last documented commit are identified. Ancestors are marked affected when a descendant is affected. +3. **Hierarchical synthesis** — `synthesizeChildren` processes child directories recursively, synthesizing subsystem summaries before individual files. `synthesizeNode` reduces all components for a directory into a final summary, writes it to `modules/.md`, and persists the cache. +4. **Text reduction** — `reduceInChunks` splits oversized items with an 800-character overlap, batches remaining items, and reduces each batch. If total output reaches 95% of input, reduction stops and results are concatenated. +5. **Fact extraction** — `extractFileFacts` invokes the LLM for each extraction step per chunk of a file. Strips outer markdown fences, filters "none" facts, and caches results keyed by file SHA-256. + +## LLM Abstraction + +The `llmCaller` interface abstracts LLM invocation with `CallLLM(ctx, systemPrompt, userPrompt) (string, error)` and `NumCtx() int`. The default implementation serializes requests to the Ollama `/api/chat` endpoint with a 10-minute timeout. On 200 OK, the full response body is read; on non-200, up to 1024 bytes of the error body are captured. + +## Metadata Cache + +`MetadataCache` persists file state and configuration across engine runs. It stores a version, a `stepsHash` derived from model ID and all prompt templates, per-file SHA-256 hashes with associated facts, and module-to-path mappings. The cache is stored as `.metadata.json` in the docs directory. If the version is incompatible, it resets. + +## Error Handling + +Errors propagate via return values. No panics or crashes are present. The cobra framework displays returned errors and exits with a non-zero status code. Specific cases: file read failures during extraction are swallowed and logged via `onStatus`; stale cleanup failures are silently ignored; lock acquisition and cache save failures are fatal. + +## Filesystem Safety + +All filesystem writes use atomic semantics: `WriteFileAtomic` writes to a temporary file first, then renames to the target path. `WriteFileSafely` uses atomic write semantics. `RemoveFileSafely` ignores `os.IsNotExist` errors. `tools.WriteFileAtomic` is used for config persistence. + +## Git Integration + +`VerifyGitRepo(repoRoot)` verifies the repository root resides inside a git repository by executing `git rev-parse --is-inside-work-tree`. `EnsureGitignoreHasLockfile` ensures `.code-reducer.lock` is included in `.gitignore`. `LoadGitignore` loads patterns for ignore pattern matching during file discovery. + +## State and Concurrency + +No synchronization mechanisms are present. Global variables (`initCmd`, `setupCmd`, `updateCmd`) are declared at package scope and initialized once; they are never modified after initialization. No mutexes, locks, atomic types, or async patterns are used. \ No newline at end of file diff --git a/wiki/modules/README.md b/wiki/modules/README.md index 9aa71e4..1cf5274 100644 --- a/wiki/modules/README.md +++ b/wiki/modules/README.md @@ -1,80 +1,509 @@ -# Code-Reducer System Architecture Reference +# Code-Reducer: Technical Documentation -## System Overview -Code-Reducer is a command-line tool designed to generate structured developer documentation (wikis) from codebases by utilizing local Large Language Models (LLMs) via Ollama. The system employs a Map-Reduce engine to perform hierarchical synthesis, transitioning from individual file analysis to high-level architectural documentation. It supports incremental updates by tracking file state via SHA256 fingerprints. +## Module Responsibility + +Code-Reducer is a CLI-driven documentation agent that automatically generates and maintains project wikis. The system orchestrates an LLM-powered synthesis pipeline across four operational modes: **init** (first-time documentation build), **setup** (interactive configuration wizard), **update** (incremental wiki regeneration), and **update** (incremental wiki regeneration). All external behavior is delegated to the `cmd` package via `cmd.RootCmd.Execute()`. + +--- + +## Data Flow + +``` +CLI Invocation + │ + ▼ +RootCmd (root.go) + │ + ├── ModeInit ──► initCmd ──► executeCommand(ModeInit) ──► engine.Run(ModeInit) + │ + ├── ModeUpdate ──► updateCmd ──► executeCommand(ModeUpdate) ──► engine.Run(ModeUpdate) + │ + └── ModeSetup ──► setupCmd ──► RunSetupFlow ──► config.SaveConfig +``` --- -## CLI Orchestration (`cmd` module) +## Subsystem: cmd + +### Command Registration + +#### init.go + +The `init` command is registered at package scope via a global `*cobra.Command` variable. On execution, it delegates to `executeCommand` with `engine.ModeInit`. The engine mode triggers a repository scan and generates the first set of wiki markdown pages. No public API surface is exposed from this file; all identifiers are unexported. + +#### root.go + +The root command (`RootCmd`) serves as the CLI application's top-level entry point. It is initialized once and never modified after creation. The root command binds two persistent flags (`modelIDFlag`, `numCtxFlag`) and provides the following unexported functions: -### Responsibility -The `cmd` module acts as the orchestration layer, managing the CLI lifecycle, environment validation, and dispatching execution to the underlying engine. It handles user interaction for initial configuration and enforces state transitions between project initialization and incremental updates. +- **`init()`** — Initializes the root command and registers all subcommands. +- **`executeCommand(mode engine.RunMode)`** — Central dispatch function. Accepts a `engine.RunMode` and returns `error`. Aggregates errors from `os.Getwd()`, `tools.VerifyGitRepo`, `checkAndRunSetup`, `config.ResolveConfig`, `checkInitStatus`, and `runEngine`. +- **`checkAndRunSetup(repoRoot string)`** — Returns `error` when the configuration file is missing and stdin is not a terminal. +- **`checkInitStatus(repoRoot, docsDir string, mode engine.RunMode)`** — Returns `error` when the requested init/update mode conflicts with the project's initialization state. +- **`runEngine(repoRoot string, cfg *config.Config, mode engine.RunMode)`** — Wraps engine errors with `fmt.Errorf("documentation run failed: %w", err)`. -### Data Flow -1. **Input Acquisition**: Captures CLI arguments and interactive user input via `os.Stdin`. -2. **Environment Validation**: Verifies the existence of a git repository and identifies the working directory. -3. **State Assessment**: Evaluates the current project state (e.g., via `engine.IsInitialized`) to enforce valid modes (`ModeInit` vs. `ModeUpdate`). -4. **Configuration Resolution**: Loads and merges settings from disk and environmental sources. -5. **Execution Dispatch**: Passes the validated `config.Config` and execution `mode` to the `engine` package. -6. **Output Stream**: Directs status updates and errors to `stdout` and `stderr`. +#### setup.go -### Core Components -* **`RootCmd`**: The primary `cobra.Command` entry point. -* **`executeCommand`**: The central controller that orchestrates directory resolution, git verification, and configuration loading. -* **`RunSetupFlow`**: An interactive process that captures LLM Model IDs, Ollama endpoints, context sizes, and documentation directory paths. -* **Subcommands**: - * `init`: Triggers a full repository scan and initial Markdown generation. - * `update`: Triggers incremental analysis by comparing current file states against the last documented git commit. +The `setup` command (`setupCmd`) implements an interactive CLI setup wizard. It generates or updates a `.code-reducer.yaml` configuration file by collecting the following values from the user: + +1. LLM Model ID +2. Ollama Base URL +3. Ollama Context Size (numeric validation with default fallback) +4. Ignore patterns (comma-separated list; supports "clear"/"none" to reset) +5. Documentation directory + +The flow loads any existing configuration from the current working directory (falling back to an empty `config.Config{}` if the load fails), applies defaults, prompts sequentially, and persists the result via `config.SaveConfig`. + +#### update.go + +The `update` command (`updateCmd`) performs incremental documentation updates. It identifies files modified since the last documented commit using cached SHA-256 hashes and regenerates the corresponding wiki pages. The operation is tied to the last documented commit as the baseline. No direct I/O is performed in this file; all logic is delegated to `internal/engine` via `executeCommand(engine.ModeUpdate)`. + +### Error Propagation + +All functions in `cmd` return `error` type. Errors propagate up the call chain via return values. No panics or crashes are present. The cobra framework displays returned errors to the user and exits with a non-zero status code. Errors from `executeCommand` are returned directly to the cobra command framework via `RunE`. + +### State and Concurrency + +No synchronization mechanisms are present. Global variables (`initCmd`, `setupCmd`, `updateCmd`) are declared at package scope and initialized once; they are never modified after initialization. No mutexes, locks, atomic types, or async patterns are used. --- -## Configuration Management (`internal/config`) +## Subsystem: internal + +### Configuration Subsystem (`internal/config`) + +#### Configuration Persistence (`io.go`) + +**Public API** + +| Function | Parameters | Return Types | +|---|---|---| +| `ConfigExists` | `cwd string` | `bool` | +| `LoadConfig` | `cwd string` | `*Config`, `error` | +| `SaveConfig` | `cwd string`, `cfg *Config` | `error` | + +**Internal Functions** + +| Function | Parameters | Return Types | +|---|---|---| +| `getConfigPath` | `cwd string` | `string` | +| `formatYAML` | `data []byte` | `string` | + +**File System Interactions** + +- **Disk reads:** `os.Stat` in `ConfigExists`; `os.ReadFile` in `LoadConfig`. +- **Disk writes:** `tools.WriteFileAtomic` in `SaveConfig`. +- No network, database, or API interactions. + +**Error Propagation** + +- `LoadConfig` returns `(*Config, error)`. If `os.ReadFile` fails, the raw error is returned. If YAML parsing fails, the error is wrapped with `fmt.Errorf("failed to parse yaml config: %w", err)`. +- `SaveConfig` returns `error`. If `yaml.Marshal` fails, the error is wrapped with `fmt.Errorf("failed to marshal yaml: %w", err)` before being passed to `tools.WriteFileAtomic`. +- `ConfigExists` derives a boolean from `os.Stat` error check (`err == nil`). +- `getConfigPath` and `formatYAML` have no error handling; failures result in panics. + +#### Configuration Resolution (`resolve.go`) + +**Public API** + +| Function | Parameters | Return Type | +|---|---|---| +| `ApplyDefaults` | `cfg *Config` | (none) | +| `ResolveConfig` | `repoRoot string`, `modelIDFlag string`, `numCtxFlag string` | `(*Config, error)` | + +**Internal Functions** + +| Function | Parameters | Return Type | +|---|---|---| +| `deduplicate` | `a []string` | `[]string` | +| `loadBaseConfig` | `repoRoot string` | `(*Config, error)` | +| `resolveModelID` | `cfgVal string`, `flagVal string` | `string` | +| `resolveBaseURL` | `cfgVal string` | `string` | +| `parsePositiveInt` | `val string`, `errMsg string` | `(int, error)` | +| `resolveNumCtx` | `cfgVal int`, `flagVal string` | `(int, error)` | + +**External I/O Interactions** + +- **Disk I/O:** `loadBaseConfig` calls `LoadConfig(repoRoot)` from `io.go`. If the file does not exist (`os.ErrNotExist`), an empty `Config{}` is returned instead of an error. Other read errors are wrapped and returned. +- **Environment Variables:** Three environment variables are read via `os.Getenv`: + - `CodeReducerModelIDEnvKey` in `resolveModelID` + - `OllamaBaseURLEnvKey` in `resolveBaseURL` + - `OllamaNumCtxEnvKey` in `resolveNumCtx` +- No network, database, or API calls. + +**Error Propagation** + +- `ResolveConfig` returns errors from `loadBaseConfig` (which wraps non-`os.ErrNotExist` errors from `LoadConfig`) and from `resolveNumCtx` (which wraps errors from `parsePositiveInt`). +- `ApplyDefaults` and `deduplicate` swallow all errors silently. +- `resolveModelID` and `resolveBaseURL` return values without errors; callers handle the empty-string case implicitly. +- `parsePositiveInt` returns a formatted error string via `fmt.Errorf`. +- Errors are propagated via Go's `error` return convention; no panics or exceptions are used. + +#### Configuration Schema (`config.go`) -### Responsibility -This module manages the hierarchical resolution, validation, and persistence of system settings required for the LLM pipeline. +**Constants** -### Data Flow -1. **Ingestion**: Aggregates data from `.code-reducer.yaml`, environment variables, and CLI flags. -2. **Resolution**: Resolves values based on the following precedence: CLI flags $\rightarrow$ Environment variables $\rightarrow$ Configuration file $\rightarrow$ System defaults. -3. **Validation**: Ensures numerical parameters (e.g., `OllamaNumCtx`) are integers greater than zero. -4. **Persistence**: Serializes the `Config` struct to YAML and performs an atomic write using a temporary file and rename pattern. +| Constant | Value | +|---|---| +| `ConfigFileName` | `.code-reducer.yaml` | +| `configFilePerm` | `0600` | +| `OllamaDefaultBaseURL` | `http://localhost:11434` | +| `OllamaDefaultModelID` | `ornith:9b` | +| `OllamaDefaultNumCtx` | `8192` | +| `DefaultDocsDir` | `wiki` | +| `CodeReducerModelIDEnvKey` | `CODE_REDUCER_MODEL_ID` | +| `OllamaBaseURLEnvKey` | `OLLAMA_BASE_URL` | +| `OllamaNumCtxEnvKey` | `OLLAMA_NUM_CTX` | -### Data Structures -* **`Config`**: The primary schema containing `ModelID`, `OllamaBaseURL`, `OllamaNumCtx`, `DocsDir`, and prompt templates for different synthesis stages (Module, Architecture, File Fact Consolidation). -* **`ExtractionStep`**: Defines specific phases of the LLM extraction pipeline, mapping a `Name` to a specific `Prompt`. +**Default Prompts** + +The following multi-line string constants define system prompts for downstream synthesis steps: + +- `DefaultSystemPrompt` — Code-Reducer role and rules (extract facts, no speculation, no code blocks) +- `DefaultModuleSynthesisPrompt` — Folder summary generation +- `DefaultArchitecturePrompt` — Global architecture document generation +- `DefaultQuickstartPrompt` — Developer quickstart guide generation +- `DefaultFileFactConsolidationPrompt` — Fact consolidation and deduplication + +**Structs** + +| Type | Fields | +|---|---| +| `ExtractionStep` | `Name` string, `Prompt` string | +| `Config` | `ModelID` string, `OllamaBaseURL` string, `OllamaNumCtx` int, `DocsDir` string, `SystemPrompt` string, `ModuleSynthesisPrompt` string, `ArchitecturePrompt` string, `QuickstartPrompt` string, `FileFactConsolidationPrompt` string, `ExtractionSteps` []ExtractionStep, `Ignore` []string | + +**Default Extraction Steps** + +`DefaultExtractionSteps` is a `[]ExtractionStep` containing: `API_SIGNATURES`, `BUSINESS_LOGIC`, `STATE_AND_CONCURRENCY`, `ERRORS_AND_SIDE_EFFECTS`. + +#### Resolution Priority Ordering + +The following priority ordering is enforced by `ResolveConfig` and its helpers: + +| Field | Priority Order | +|---|---| +| `ModelID` | CLI flag → `CODE_REDUCER_MODEL_ID` env var → config file value | +| `OllamaBaseURL` | `OLLAMA_BASE_URL` env var → config file value | +| `OllamaNumCtx` | CLI flag → `OLLAMA_NUM_CTX` env var → config file value | +| `DocsDir` | System default (`wiki`) when empty/zero | +| `ExtractionSteps` | System default when empty | +| All prompt fields | System default when empty | + +`DocsDir` is additionally trimmed and path-cleaned during default application. --- -## Map-Reduce Documentation Engine (`internal/engine`) +## Engine Subsystem (`internal/engine`) + +### Core Infrastructure + +#### LLM Client (`client.go`) + +The `llmCaller` interface abstracts LLM invocation. The unexported `llmClient` type implements it by serializing requests to the Ollama `/api/chat` endpoint and returning text responses. + +```go +type llmCaller interface { + CallLLM(ctx context.Context, systemPrompt, userPrompt string) (string, error) + NumCtx() int +} +``` + +`CallLLM` sends an HTTP POST with a 10-minute timeout. On a 200 OK, the full response body is read; on non-200, up to 1024 bytes of the error body are captured. Marshaling, request creation, and response parsing errors are wrapped with `fmt.Errorf`. The `onStatus` callback fires when `done_reason == "length"`. + +#### Metadata Cache (`cache.go`) + +`MetadataCache` persists file state and configuration across engine runs. It stores a version, a `stepsHash` derived from model ID and all prompt templates, per-file SHA-256 hashes with associated facts, and module-to-path mappings. + +```go +type FileCacheEntry struct { + SHA256 string + Facts string +} + +type MetadataCache struct { + Version int + StepsHash string + Files map[string]FileCacheEntry + Modules map[string]string +} +``` + +`loadMetadataCache` reads `.metadata.json` from `docsDir`. If absent, it returns an empty cache; if the version is incompatible, it resets. `saveMetadataCache` writes indented JSON. `IsInitialized` checks file existence. `computeSHA256` hashes arbitrary file paths. + +### Tree Construction & Change Detection + +#### Directory Tree (`tree.go`) + +`DirNode` represents a directory containing files and child directories. `buildTree` parses a file slice into a hierarchical `*DirNode` structure. + +```go +type DirNode struct { + Path string + Files []string + Children map[string]*DirNode +} +``` + +`FileChange` tracks a file's status (`StatusAdded`, `StatusModified`, `StatusDeleted`) and path. `propagateAffected` walks the tree upward, marking ancestors affected when a descendant is affected. `isNodeAffected` checks whether a node's files appear in the changed set or its module is missing from cache. `determineAffected` combines changed files with cache state to produce a final affected-dirs map. + +#### Orchestrator (`orchestrator.go`) + +The `orchestrator` struct holds the `llmCaller` and coordinates the full pipeline. `pipelineState` is created fresh per invocation and contains `repoRoot`, `cfg`, `cache`, `affectedDirs`, `precalculatedHashes`, and `onStatus`. + +```go +type orchestrator struct { + client llmCaller +} + +type pipelineState struct { + client llmCaller + repoRoot string + cfg *config.Config + cache *MetadataCache + affectedDirs map[string]bool + precalculatedHashes map[string]string + onStatus StatusFunc +} +``` + +`Run` is the entry point. It acquires a lock via `security.AcquireLock`, loads `.gitignore`, loads the metadata cache, and dispatches to `RunInit` or `RunUpdate`. `RunInit` builds the tree, hashes all code files, synthesizes the root summary, generates `architecture.md` and `quickstart.md`, updates `AGENTS.md`, and persists the cache. `RunUpdate` detects file changes, prunes stale cache entries, and regenerates only affected modules. + +`setupPipeline` initializes the cache and affected-dirs map. `teardownPipeline` cleans up stale module files and empty directories. `hashCodeFiles` computes SHA-256 hashes for a file list. `collectTreeDirs` returns the set of directories present in the tree. `updateAgentGuidelines` appends documentation references to `AGENTS.md`. + +### Hierarchical Synthesis + +#### Synthesis Pipeline (`synthesize.go`) + +`extractFileFacts` invokes the LLM for each extraction step per chunk of a file. It strips outer markdown fences, filters "none" facts, and caches results keyed by file SHA-256. Read failures are swallowed via `onStatus`; LLM and chunking failures propagate. + +`synthesizeChildren` processes child directories recursively, synthesizing subsystem summaries before individual files. `collectComponents` combines subsystem summaries and file facts into a unified component list. `synthesizeNode` reduces all components for a directory into a final summary, writes it to `modules/.md`, and persists the cache. If no components are produced, the cached summary is deleted and the markdown file is removed. + +```go +func extractFileFacts(ctx context.Context, p *pipelineState, f string, siblingFiles []string) (string, error) +func synthesizeChildren(ctx context.Context, p *pipelineState, node *DirNode) ([]string, error) +func collectComponents(ctx context.Context, p *pipelineState, node *DirNode, subsystems []string) ([]string, error) +func synthesizeNode(ctx context.Context, p *pipelineState, node *DirNode) (string, error) +``` + +`buildExtractionPrompt` constructs the user prompt for a single extraction step. `maxExtractionOverhead` computes the maximum characters available for extraction given system prompt and file context. `toSafeMarkdownFilename` sanitizes a module path for use as a filename. + +#### Text Reduction & Chunking (`chunking.go`) -### Responsibility -The engine performs the core logic of transforming source code into structured documentation through a hierarchical reduction process. It manages file discovery, change detection, and recursive LLM-based content reduction. +`reduceWithLLM` calls the LLM to reduce a batch of items. `reduceInChunks` splits oversized items with an 800-character overlap, batches remaining items, and reduces each batch. If total output reaches 95% of input, reduction stops and results are concatenated. Otherwise, the intermediate results are recursively reduced. -### Data Flow -1. **Acquisition**: The `Runner` secures a repository lock. -2. **Discovery**: Files are mapped into a hierarchical `DirNode` tree structure. -3. **Change Detection**: The `Orchestrator` compares current file SHA256 hashes against the `MetadataCache`. "Affected" status propagates up the `DirNode` tree. -4. **Hierarchical Synthesis**: - * **Leaf Processing**: Files are processed via `Chunking`. If content exceeds the context window (using a 0.75 `contextWindowAllocRatio`), recursive reduction is applied. - * **Aggregation**: Extracted information is aggregated from the file level up to the module and directory levels. -5. **Persistence**: Synthesized Markdown and `MetadataCache` (JSON) are written to the `docsDir`. +`chunkTextWithOverlap` splits text with overlap validation. `stripOuterMarkdownFence` removes surrounding code fences. `calculateFileLimit` computes the maximum processing characters from context window minus system prompt overhead. `isNoneFact` filters empty, "none", and "n/a" facts. `expandOversizedItems` splits items exceeding the limit. `batchItems` groups remaining items. `countRunes` counts Unicode code points. `reduceItems` orchestrates the reduction with a custom function. -### Core Components -* **`Synthesizer`**: Traverses the `DirNode` tree to decide between using cached summaries or triggering new LLM extractions. -* **`Chunking`**: Implements recursive reduction for oversized segments. Recursion terminates if the output size is $\ge$ 95% of the input. -* **`MetadataCache`**: Persists SHA256 fingerprints to facilitate incremental updates. -* **`LLM Client`**: A synchronous HTTP client communicating via the `api/chat` protocol. +### Error Handling Summary + +| Operation | Failure Mode | Handling | +|-----------|-------------|----------| +| Lock acquisition | `security.AcquireLock` fails | Fatal error returned | +| Gitignore load | `tools.LoadGitignore` fails | Fatal error returned | +| Code discovery | `tools.DiscoverCodeFiles` fails | Fatal error returned | +| File read (extraction) | `tools.ReadFileSafely` fails | Swallowed, logged via `onStatus` | +| LLM call | `CallLLM` fails | Propagated as wrapped error | +| Chunking | Invalid `maxRunes` or `overlapRunes` | Returned as error | +| Disk write | `tools.WriteFileSafely` fails | Fatal error returned | +| Stale cleanup | `os.Remove` fails | Silently ignored | +| Cache save | Marshaling or write fails | Fatal error returned | +| HTTP non-200 | Response body read | Wrapped with status code | +| Context cancellation | Any operation | Propagated as error | --- -## External Tooling Configurations +## Security Subsystem (`security.go`) + +### Module Responsibility + +This module provides two non-negotiable security primitives for a repository-level tool: + +1. **Path traversal prevention** — `SafeResolve` ensures any input path resolves strictly inside the repository root, even when symlinks are involved. +2. **Process-level locking** — `AcquireLock` / `SimpleLock` / `Unlock` prevents concurrent execution by multiple processes of the same tool. + +**Data flow:** `SafeResolve(repoRoot, inputPath)` → `(string, error)`. `AcquireLock(repoRoot)` → `(*SimpleLock, error)`. `(*SimpleLock).Unlock()` → `error`. No global mutable state exists; all constants and sentinel errors are immutable. + +### Path Traversal Prevention + +#### `SafeResolve(repoRoot, inputPath string) (string, error)` + +Cleans `inputPath` and verifies the resolved result lies strictly inside the repository root. + +**Algorithm:** + +1. `filepath.Abs(repoRoot)` — canonicalize the repository root to an absolute path. +2. `filepath.EvalSymlinks(repoRoot)` — resolve symlinks on the repository root itself. +3. Walk up the input path from the root, stopping at the first physically existing ancestor (checked via `os.Lstat`). +4. `filepath.EvalSymlinks` on that existing ancestor. +5. Rebuild the full path by appending the non-existent suffix components. +6. `filepath.Rel` computes the relative path from the resolved root to the rebuilt path. If the result is not `""`, return `ErrPathTraversal`. + +**Error propagation:** Returns `(string, error)`. Errors are wrapped with `fmt.Errorf` using `%w` for chainability. `ErrPathTraversal` is returned directly as a sentinel. + +### Process-Level Locking + +#### `AcquireLock(repoRoot string) (*SimpleLock, error)` + +Acquires a simple file lock in `repoRoot` using `O_EXCL` for atomicity. + +**Algorithm:** + +1. Resolve the lock file path using `SafeResolve`. +2. `os.OpenFile` with `O_WRONLY|O_CREATE|O_EXCL`. If the file already exists, return `ErrLockHeld`. +3. Write the current process PID (`os.Getpid`) to the file. +4. Return a `*SimpleLock` handle. + +**Error propagation:** If `OpenFile` fails with `os.IsExist`, wraps `ErrLockHeld`. If `Write` fails, the file is closed and removed before returning a wrapped error. + +#### `(*SimpleLock) Unlock() error` + +Releases the lock by closing the file and removing it. Idempotent and thread-safe. + +**Algorithm:** + +1. Acquire the internal `sync.Mutex` (`l.mu`). +2. If `l.closed` is true, return `nil`. +3. Close the file descriptor. +4. Remove the lock file. If both close and remove fail, return the error from the non-nil one. + +**Error propagation:** If close succeeds but remove fails (and is not `os.IsNotExist`), the remove error is assigned to `err`. If close fails, the remove error is ignored. + +### Types + +#### `SimpleLock` + +```go +type SimpleLock struct { + lockPath string + file *os.File + mu sync.Mutex + closed bool +} +``` + +- `lockPath` — absolute path to the lock file. +- `file` — open file descriptor for the lock. +- `mu` — protects `closed` field. +- `closed` — set to `true` in `Unlock()`; guards idempotency. + +### Constants and Errors + +**Constants** + +- `LockFileName` = `".code-reducer.lock"` +- `defaultFilePerm` = `0644` + +**Error Variables** + +- `ErrPathTraversal` = `errors.New("security violation: path traversal detected")` +- `ErrLockHeld` = `errors.New("lock is already held by another process")` + +### External I/O Interactions + +All operations target the local filesystem. No network, database, or API interactions. + +| Operation | Purpose | +|-----------|---------| +| `filepath.Abs` | Resolves absolute path for `repoRoot` | +| `filepath.EvalSymlinks` | Resolves symlinks on `repoRoot` and intermediate ancestors | +| `os.Lstat` | Checks existence of path components during traversal resolution | +| `os.OpenFile` | Opens the lock file with `O_WRONLY|O_CREATE|O_EXCL` | +| `os.Remove` | Removes the lock file during `Unlock()` and on write failure in `AcquireLock()` | +| `os.Getpid` | Reads process ID (no external I/O) | +| `filepath.Rel` | Computes relative path to verify containment | + +### Synchronization + +- `SimpleLock.Unlock()` uses `sync.Mutex` (`l.mu`) to protect the `closed` field. +- `AcquireLock()` is a package-level function that creates a new `SimpleLock` instance; it does not use any synchronization mechanism. + +### Error Handling Summary + +- Errors are not logged or printed; they propagate to the caller. +- No `panic`/`recover` is used. +- Sentinel errors (`ErrPathTraversal`, `ErrLockHeld`) are returned directly. +- All other errors are wrapped with `fmt.Errorf` using `%w` for chainability. + +--- + +## Tools Subsystem (`internal/tools`) + +### Module Responsibility + +This module provides safe filesystem operations and repository verification utilities for source code management workflows. It operates on a repository root (`repoRoot`) and resolves virtual paths through a security layer to prevent path traversal attacks. All operations are disk I/O only—no network, database, or external API calls. + +**Data Flow:** + +``` +caller → virtualPath → security.SafeResolve → filesystem path → os.* operations +``` + +All public functions accept a `repoRoot` parameter and resolve virtual paths before accessing the filesystem. Errors are propagated as return values; no panics occur. + +### File Tools (`file_tools.go`) + +#### Safe File Access + +**`FileExistsSafely(repoRoot string, virtualPath string) bool`** + +Checks whether a file exists at the resolved filesystem path. Swallows `os.Stat` errors and returns `false` on any failure. + +**`ReadFileSafely(repoRoot string, virtualPath string) ([]byte, error)`** + +Reads file contents from the resolved filesystem path. Returns the file data and an error. Swallows `os.Open` errors. + +**`RemoveFileSafely(repoRoot string, virtualPath string) error`** + +Removes a file at the resolved filesystem path. Ignores `os.IsNotExist` errors (file may already be absent). Propagates all other errors. + +#### Atomic File Writes + +**`WriteFileAtomic(targetPath string, data []byte, perm os.FileMode) error`** + +Writes data to `targetPath` atomically: writes to a temporary file first, then renames to the target path. Wraps all sub-errors with `fmt.Errorf("failed to %s: %w", ...)`. + +**`WriteFileSafely(repoRoot string, virtualPath string, content []byte) error`** + +Writes content to the resolved filesystem path. Uses atomic write semantics. + +#### Gitignore Integration + +**`EnsureGitignoreHasLockfile(repoRoot string) error`** + +Ensures `.code-reducer.lock` is included in `.gitignore` to prevent it from being committed. + +**`LoadGitignore(repoRoot string) ([]string, error)`** + +Loads `.gitignore` patterns from the repository root. Returns the parsed patterns and an error. + +**`ShouldIgnoreFile(relPath string, gitIgnore *ignore.GitIgnore) bool`** + +Determines whether a relative path matches any loaded gitignore patterns. Uses `ignore.CompileIgnoreLines` for pattern parsing (in-memory, no external I/O). + +### Global State + +| Variable | Type | Mutability | +|---|---|---| +| `skippedFileNames` | `map[string]bool` | Immutable after init | +| `skippedExtensions` | `map[string]bool` | Immutable after init | +| `skippedTestSuffixes` | `[]string` | Immutable after init | + +These maps are initialized once and never modified. They define the multi-layer file filtering rules applied during file discovery. + +### Git Tools (`git_tools.go`) + +#### Repository Verification + +**`VerifyGitRepo(repoRoot string) error`** + +Verifies whether `repoRoot` resides inside a git repository by executing `git rev-parse --is-inside-work-tree` in the specified directory. + +**Process Execution Details:** + +- Spawns `git` as a child process via `exec.Command("git", "--no-pager", "rev-parse", "--is-inside-work-tree")` +- Redirects child process stderr to a `bytes.Buffer` for capture +- Sets child process working directory to `repoRoot` -### `mkdocs.yml` -Defines the deployment parameters for the generated documentation wiki. -* **Domain Logic**: Orchestrates the Map-Reduce engine and local LLM interaction. -* **External I/O**: Fetches frontend assets (JS/CSS) from `cdn.jsdelivr.net` and `cdnjs.cloudflare.com`. +**Error Handling:** -### `sonar-project.properties` -Configures static analysis and code coverage reporting. -* **Filtering Logic**: Excludes test files, dependencies, and auxiliary assets from source analysis. -* **I/O**: Reads from the current directory and the `coverage.out` report. \ No newline at end of file +- If `cmd.Run()` returns an error, the function wraps it with `fmt.Errorf` and returns the wrapped error (including captured stderr content) +- If `cmd.Run()` returns nil, returns nil (success) +- No panics or swallowed errors \ No newline at end of file diff --git a/wiki/modules/cmd/README.md b/wiki/modules/cmd/README.md index 3ed841f..70394ca 100644 --- a/wiki/modules/cmd/README.md +++ b/wiki/modules/cmd/README.md @@ -1,73 +1,60 @@ -# Module: `cmd` +# cmd Package Architecture -## Module Overview -The `cmd` module serves as the command-line interface (CLI) orchestration layer. It manages the lifecycle of project documentation generation by coordinating user input, configuration persistence, git repository validation, and the execution of the underlying documentation engine. +## Module Responsibility + +The `cmd` package implements the CLI entry point for a documentation agent that writes and maintains project wikis. It orchestrates the documentation engine across three operational modes: **init** (first-time documentation build), **setup** (interactive configuration wizard), and **update** (incremental wiki regeneration). All commands are registered as `*cobra.Command` instances and share a common execution path through `executeCommand`, which dispatches to the engine based on the requested `engine.RunMode`. ## Data Flow -1. **Input Acquisition**: CLI arguments are parsed via `RootCmd`. User-provided flags (`--model-id`, `--num-ctx`) and interactive input (`os.Stdin`) are captured. -2. **Environment Validation**: The module verifies the presence of a git repository and determines the current working directory via `os.Getwd`. -3. **State Assessment**: The module checks for existing configuration files and determines if the project is in an initialized state to validate the requested operation (`init` vs. `update`). -4. **Configuration Resolution**: Configuration is loaded from disk, merged with defaults, and optionally updated via an interactive setup flow. -5. **Execution Dispatch**: The validated configuration and mode are passed to the `engine` package to perform documentation generation or incremental updates. -6. **Output Stream**: Results, status updates, and errors are streamed to `stdout` and `stderr`. - -## Core Components - -### Command Orchestration (`root.go`) -The central logic for the CLI lifecycle is contained within this file. - -* **`RootCmd`** (`*cobra.Command`): The entry point for the CLI application. -* **`executeCommand(mode engine.Mode) error`**: The primary controller. It performs the following sequence: - * Resolves the working directory. - * Verifies the git repository status. - * Triggers `checkAndRunSetup` if configuration is missing and `os.Stdin` is a TTY. - * Resolves the final configuration. - * Validates that the current project state matches the requested `engine.Mode`. - * Dispatches execution to `runEngine`. -* **`checkAndRunSetup(repoRoot string) error`**: Evaluates if `RunSetupFlow` is required based on the existence of configuration files and terminal availability. -* **`checkInitStatus(repoRoot, docsDir string, mode engine.Mode) error`**: Enforces state transition rules: - * `ModeInit` is prohibited if the project is already initialized. - * `ModeUpdate` is prohibited if the project is not yet initialized. -* **`runEngine(repoRoot string, cfg *config.Config, mode engine.Mode) error`**: Orchestrates engine execution with signal handling for `os.Interrupt` and `syscall.SIGTERM`. - -### Command Definitions (`init.go`, `update.go`) -These files act as thin wrappers that map CLI subcommands to specific engine modes. - -* **`init` subcommand**: Dispatches `engine.ModeInit` to trigger the initial scanning of the repository and generation of wiki markdown files. -* **`update` subcommand**: Dispatches `engine.ModeUpdate` to perform incremental updates by comparing current file states against the last documented git commit. - -### Configuration Lifecycle (`setup.go`) -Handles the interactive provisioning of project settings. - -* **`RunSetupFlow(repoRoot string) error`**: Manages a stateful, interactive session to collect: - * LLM Model ID. - * Ollama Base URL. - * Ollama Context Size. - * File/directory ignore patterns. - * Documentation directory paths. -* **Persistence**: Finalized settings are consolidated into a `config.Config` object and written to disk via `config.SaveConfig`. - -## State and Side Effects - -### Mutable State -* **Global Flags**: `modelIDFlag` and `numCtxFlag` are modified during the `init()` phase and persist for the lifetime of the process. -* **Command State**: `RootCmd` state is modified via `.SetArgs()` and `.Execute()` during testing. - -### External I/O -* **Filesystem**: - * Directory/file creation via `os.MkdirAll` and `os.WriteFile`. - * Configuration persistence via `config.SaveConfig` and `config.LoadConfig`. - * Git verification via `tools.VerifyGitRepo`. - * State check via `engine.IsInitialized`. -* **Standard Streams**: - * `os.Stdin`: Used for interactive parameter collection. - * `os.Stdout`: Used for status messages and engine events. - * `os.Stderr`: Used for error reporting. - -### Error Handling -* **Propagation**: Errors are passed up the call stack via return values. -* **Wrapping**: `os.Getwd` and `runner.Run` errors are wrapped with context using `%w`. -* **Swallowed Errors**: - * `config.LoadConfig` errors are swallowed in `RunSetupFlow` (defaults are returned instead). - * `strconv.Atoi` errors in `promptContextSize` are swallowed (default values are returned instead). - * `RootCmd.Execute()` return values are swallowed in `cmd_test.go`. \ No newline at end of file + +``` +CLI Invocation + │ + ▼ +RootCmd (root.go) + │ + ├── ModeInit ──► initCmd ──► executeCommand(ModeInit) ──► engine.Run(ModeInit) + │ + ├── ModeUpdate ──► updateCmd ──► executeCommand(ModeUpdate) ──► engine.Run(ModeUpdate) + │ + └── ModeSetup ──► setupCmd ──► RunSetupFlow ──► config.SaveConfig +``` + +## Command Registration + +### init.go + +The `init` command is registered at package scope via a global `*cobra.Command` variable. On execution, it delegates to `executeCommand` with `engine.ModeInit`. The engine mode triggers a repository scan and generates the first set of wiki markdown pages. No public API surface is exposed from this file; all identifiers are unexported. + +### root.go + +The root command (`RootCmd`) serves as the CLI application's top-level entry point. It is initialized once and never modified after creation. The root command binds two persistent flags (`modelIDFlag`, `numCtxFlag`) and provides the following unexported functions: + +- **`init()`** — Initializes the root command and registers all subcommands. +- **`executeCommand(mode engine.RunMode)`** — Central dispatch function. Accepts a `engine.RunMode` and returns `error`. Aggregates errors from `os.Getwd()`, `tools.VerifyGitRepo`, `checkAndRunSetup`, `config.ResolveConfig`, `checkInitStatus`, and `runEngine`. +- **`checkAndRunSetup(repoRoot string)`** — Returns `error` when the configuration file is missing and stdin is not a terminal. +- **`checkInitStatus(repoRoot, docsDir string, mode engine.RunMode)`** — Returns `error` when the requested init/update mode conflicts with the project's initialization state. +- **`runEngine(repoRoot string, cfg *config.Config, mode engine.RunMode)`** — Wraps engine errors with `fmt.Errorf("documentation run failed: %w", err)`. + +### setup.go + +The `setup` command (`setupCmd`) implements an interactive CLI setup wizard. It generates or updates a `.code-reducer.yaml` configuration file by collecting the following values from the user: + +1. LLM Model ID +2. Ollama Base URL +3. Ollama Context Size (numeric validation with default fallback) +4. Ignore patterns (comma-separated list; supports "clear"/"none" to reset) +5. Documentation directory + +The flow loads any existing configuration from the current working directory (falling back to an empty `config.Config{}` if the load fails), applies defaults, prompts sequentially, and persists the result via `config.SaveConfig`. + +### update.go + +The `update` command (`updateCmd`) performs incremental documentation updates. It identifies files modified since the last documented commit using cached SHA-256 hashes and regenerates the corresponding wiki pages. The operation is tied to the last documented commit as the baseline. No direct I/O is performed in this file; all logic is delegated to `internal/engine` via `executeCommand(engine.ModeUpdate)`. + +## Error Propagation + +All functions in `cmd` return `error` type. Errors propagate up the call chain via return values. No panics or crashes are present. The cobra framework displays returned errors to the user and exits with a non-zero status code. Errors from `executeCommand` are returned directly to the cobra command framework via `RunE`. + +## State and Concurrency + +No synchronization mechanisms are present. Global variables (`initCmd`, `setupCmd`, `updateCmd`) are declared at package scope and initialized once; they are never modified after initialization. No mutexes, locks, atomic types, or async patterns are used. \ No newline at end of file diff --git a/wiki/modules/internal/README.md b/wiki/modules/internal/README.md index 08d4214..da98c9b 100644 --- a/wiki/modules/internal/README.md +++ b/wiki/modules/internal/README.md @@ -1,100 +1,453 @@ -# Code-Reducer System Architecture Reference +# internal — Codebase Documentation Synthesis Engine -## Subsystem: `internal/config` +## Module Responsibility -### Module Responsibility and Data Flow -The `internal/config` package manages the lifecycle, hierarchical resolution, and persistence of configuration settings. It enables the merging of settings from multiple sources into a unified `Config` struct to direct the LLM-based pipeline. +The `internal` package implements an LLM-driven pipeline that automatically generates and maintains codebase documentation. It coordinates four subsystems: -**Data Flow Pipeline:** -1. **Ingestion**: Aggregation of data from the local filesystem (`.code-reducer.yaml`), environment variables, and CLI arguments. -2. **Resolution**: Hierarchical merging based on precedence: CLI flags > Environment variables > Configuration file > System defaults. -3. **Validation**: Numerical parameters (e.g., `OllamaNumCtx`) are validated to ensure they are non-zero and valid integers. -4. **Output**: Returns a validated `*Config` pointer for LLM prompt construction and execution logic. -5. **Persistence**: Configuration changes are serialized to YAML and written via an atomic rename pattern. +- **config** — Configuration lifecycle: persistence, resolution, and default application. +- **engine** — Orchestration of the synthesis pipeline: cache management, change detection, hierarchical synthesis, and final persistence. +- **security** — Path traversal prevention and process-level locking. +- **tools** — Safe filesystem operations and repository verification utilities. -### Data Structures +**Data flow:** `ResolveConfig` (config) → `Run` (engine) → `security.SafeResolve` / `security.AcquireLock` / `tools.*` for all filesystem access. The engine reads configuration, acquires a lock, parses `.gitignore`, discovers code files, computes SHA-256 hashes, builds a directory tree, and dispatches to `RunInit` or `RunUpdate`. The orchestrator coordinates cache loading, change detection, hierarchical synthesis, and final persistence. -#### `Config` -The primary operational schema for the system. +--- + +## Configuration Subsystem (`internal/config`) + +### Configuration Persistence (`io.go`) -| Field | Type | Description | +**Public API** + +| Function | Parameters | Return Types | |---|---|---| -| `ModelID` | `string` | Identifier for the LLM model. | -| `OllamaBaseURL` | `string` | Base endpoint for the local Ollama API. | -| `OllamaNumCtx` | `int` | Context window size for the LLM. | -| `DocsDir` | `string` | Target directory for generated documentation. | -| `SystemPrompt` | `string` | Primary system message for the LLM. | -| `ModuleSynthesisPrompt` | `string` | Prompt template for module-level synthesis. | -| `ArchitecturePrompt` | `string` | Prompt template for architectural analysis. | -| `FileFactConsolidationPrompt` | `string` | Prompt template for consolidating file facts. | -| `ExtractionSteps` | `[]ExtractionStep` | Sequence of extraction phases for the pipeline. | -| `Ignore` | `[]string` | Paths or patterns excluded from analysis. | - -#### `ExtractionStep` -Defines a specific phase in the multi-step LLM extraction process. - -| Field | Type | Description | +| `ConfigExists` | `cwd string` | `bool` | +| `LoadConfig` | `cwd string` | `*Config`, `error` | +| `SaveConfig` | `cwd string`, `cfg *Config` | `error` | + +**Internal Functions** + +| Function | Parameters | Return Types | |---|---|---| -| `Name` | `string` | Identifier for the extraction phase. | -| `Prompt` | `string` | Specific prompt utilized for this phase. | - -### Hierarchical Resolution and Validation -The `ResolveConfig` function determines the final state of the `Config` instance using the following precedence: -1. **Command-line Flags**: `modelIDFlag`, `numCtxFlag`. -2. **Environment Variables**: `CODE_REDUCER_MODEL_ID`, `OLLAMA_BASE_URL`, `OLLAMA_NUM_CTX`. -3. **Configuration File**: Values from `.code-reducer.yaml`. -4. **System Defaults**: Hardcoded values. - -**Constraints:** -* **Numerical Validation**: `OllamaNumCtx` must be a valid integer and greater than zero. Non-zero/non-integer environment variables result in an error. -* **Deduplication**: The `Ignore` slice is deduplicated during resolution. -* **Default Logic**: If `ExtractionSteps` is empty in the config file, the system defaults to the `DefaultExtractionSteps` sequence. - -### I/O and Persistence -* **`ConfigExists`**: Employs `os.Stat` to check for the presence of the configuration file. All errors are swallowed; returns `false` on error. -* **`LoadConfig`**: Utilizes `os.ReadFile` and `yaml.Unmarshal`. Errors are wrapped and returned. -* **`SaveConfig`**: Implements an atomic write pattern: - 1. `yaml.Marshal` the struct to bytes. - 2. `os.CreateTemp` a temporary file in the target directory. - 3. Write data and invoke `os.Sync` for durability. - 4. `os.Chmod` to set permissions. - 5. `os.Rename` to atomically replace the target file. - * **Note**: The deferred `os.Remove` does not contain a `recover()`. +| `getConfigPath` | `cwd string` | `string` | +| `formatYAML` | `data []byte` | `string` | + +**File System Interactions** + +- **Disk reads:** `os.Stat` in `ConfigExists`; `os.ReadFile` in `LoadConfig`. +- **Disk writes:** `tools.WriteFileAtomic` in `SaveConfig`. +- No network, database, or API interactions. + +**Error Propagation** + +- `LoadConfig` returns `(*Config, error)`. If `os.ReadFile` fails, the raw error is returned. If YAML parsing fails, the error is wrapped with `fmt.Errorf("failed to parse yaml config: %w", err)`. +- `SaveConfig` returns `error`. If `yaml.Marshal` fails, the error is wrapped with `fmt.Errorf("failed to marshal yaml: %w", err)` before being passed to `tools.WriteFileAtomic`. +- `ConfigExists` derives a boolean from `os.Stat` error check (`err == nil`). +- `getConfigPath` and `formatYAML` have no error handling; failures result in panics. + +### Configuration Resolution (`resolve.go`) + +**Public API** + +| Function | Parameters | Return Type | +|---|---|---| +| `ApplyDefaults` | `cfg *Config` | (none) | +| `ResolveConfig` | `repoRoot string`, `modelIDFlag string`, `numCtxFlag string` | `(*Config, error)` | + +**Internal Functions** + +| Function | Parameters | Return Type | +|---|---|---| +| `deduplicate` | `a []string` | `[]string` | +| `loadBaseConfig` | `repoRoot string` | `(*Config, error)` | +| `resolveModelID` | `cfgVal string`, `flagVal string` | `string` | +| `resolveBaseURL` | `cfgVal string` | `string` | +| `parsePositiveInt` | `val string`, `errMsg string` | `(int, error)` | +| `resolveNumCtx` | `cfgVal int`, `flagVal string` | `(int, error)` | + +**External I/O Interactions** + +- **Disk I/O:** `loadBaseConfig` calls `LoadConfig(repoRoot)` from `io.go`. If the file does not exist (`os.ErrNotExist`), an empty `Config{}` is returned instead of an error. Other read errors are wrapped and returned. +- **Environment Variables:** Three environment variables are read via `os.Getenv`: + - `CodeReducerModelIDEnvKey` in `resolveModelID` + - `OllamaBaseURLEnvKey` in `resolveBaseURL` + - `OllamaNumCtxEnvKey` in `resolveNumCtx` +- No network, database, or API calls. + +**Error Propagation** + +- `ResolveConfig` returns errors from `loadBaseConfig` (which wraps non-`os.ErrNotExist` errors from `LoadConfig`) and from `resolveNumCtx` (which wraps errors from `parsePositiveInt`). +- `ApplyDefaults` and `deduplicate` swallow all errors silently. +- `resolveModelID` and `resolveBaseURL` return values without errors; callers handle the empty-string case implicitly. +- `parsePositiveInt` returns a formatted error string via `fmt.Errorf`. +- Errors are propagated via Go's `error` return convention; no panics or exceptions are used. + +### Configuration Schema (`config.go`) + +**Constants** + +| Constant | Value | +|---|---| +| `ConfigFileName` | `.code-reducer.yaml` | +| `configFilePerm` | `0600` | +| `OllamaDefaultBaseURL` | `http://localhost:11434` | +| `OllamaDefaultModelID` | `ornith:9b` | +| `OllamaDefaultNumCtx` | `8192` | +| `DefaultDocsDir` | `wiki` | +| `CodeReducerModelIDEnvKey` | `CODE_REDUCER_MODEL_ID` | +| `OllamaBaseURLEnvKey` | `OLLAMA_BASE_URL` | +| `OllamaNumCtxEnvKey` | `OLLAMA_NUM_CTX` | + +**Default Prompts** + +The following multi-line string constants define system prompts for downstream synthesis steps: + +- `DefaultSystemPrompt` — Code-Reducer role and rules (extract facts, no speculation, no code blocks) +- `DefaultModuleSynthesisPrompt` — Folder summary generation +- `DefaultArchitecturePrompt` — Global architecture document generation +- `DefaultQuickstartPrompt` — Developer quickstart guide generation +- `DefaultFileFactConsolidationPrompt` — Fact consolidation and deduplication + +**Structs** + +| Type | Fields | +|---|---| +| `ExtractionStep` | `Name` string, `Prompt` string | +| `Config` | `ModelID` string, `OllamaBaseURL` string, `OllamaNumCtx` int, `DocsDir` string, `SystemPrompt` string, `ModuleSynthesisPrompt` string, `ArchitecturePrompt` string, `QuickstartPrompt` string, `FileFactConsolidationPrompt` string, `ExtractionSteps` []ExtractionStep, `Ignore` []string | + +**Default Extraction Steps** + +`DefaultExtractionSteps` is a `[]ExtractionStep` containing: `API_SIGNATURES`, `BUSINESS_LOGIC`, `STATE_AND_CONCURRENCY`, `ERRORS_AND_SIDE_EFFECTS`. + +### Resolution Priority Ordering + +The following priority ordering is enforced by `ResolveConfig` and its helpers: + +| Field | Priority Order | +|---|---| +| `ModelID` | CLI flag → `CODE_REDUCER_MODEL_ID` env var → config file value | +| `OllamaBaseURL` | `OLLAMA_BASE_URL` env var → config file value | +| `OllamaNumCtx` | CLI flag → `OLLAMA_NUM_CTX` env var → config file value | +| `DocsDir` | System default (`wiki`) when empty/zero | +| `ExtractionSteps` | System default when empty | +| All prompt fields | System default when empty | + +`DocsDir` is additionally trimmed and path-cleaned during default application. + +--- + +## Engine Subsystem (`internal/engine`) + +### Core Infrastructure + +#### LLM Client (`client.go`) + +The `llmCaller` interface abstracts LLM invocation. The unexported `llmClient` type implements it by serializing requests to the Ollama `/api/chat` endpoint and returning text responses. + +```go +type llmCaller interface { + CallLLM(ctx context.Context, systemPrompt, userPrompt string) (string, error) + NumCtx() int +} +``` + +`CallLLM` sends an HTTP POST with a 10-minute timeout. On a 200 OK, the full response body is read; on non-200, up to 1024 bytes of the error body are captured. Marshaling, request creation, and response parsing errors are wrapped with `fmt.Errorf`. The `onStatus` callback fires when `done_reason == "length"`. + +#### Metadata Cache (`cache.go`) + +`MetadataCache` persists file state and configuration across engine runs. It stores a version, a `stepsHash` derived from model ID and all prompt templates, per-file SHA-256 hashes with associated facts, and module-to-path mappings. + +```go +type FileCacheEntry struct { + SHA256 string + Facts string +} + +type MetadataCache struct { + Version int + StepsHash string + Files map[string]FileCacheEntry + Modules map[string]string +} +``` + +`loadMetadataCache` reads `.metadata.json` from `docsDir`. If absent, it returns an empty cache; if the version is incompatible, it resets. `saveMetadataCache` writes indented JSON. `IsInitialized` checks file existence. `computeSHA256` hashes arbitrary file paths. + +### Tree Construction & Change Detection + +#### Directory Tree (`tree.go`) + +`DirNode` represents a directory containing files and child directories. `buildTree` parses a file slice into a hierarchical `*DirNode` structure. + +```go +type DirNode struct { + Path string + Files []string + Children map[string]*DirNode +} +``` + +`FileChange` tracks a file's status (`StatusAdded`, `StatusModified`, `StatusDeleted`) and path. `propagateAffected` walks the tree upward, marking ancestors affected when a descendant is affected. `isNodeAffected` checks whether a node's files appear in the changed set or its module is missing from cache. `determineAffected` combines changed files with cache state to produce a final affected-dirs map. + +#### Orchestrator (`orchestrator.go`) + +The `orchestrator` struct holds the `llmCaller` and coordinates the full pipeline. `pipelineState` is created fresh per invocation and contains `repoRoot`, `cfg`, `cache`, `affectedDirs`, `precalculatedHashes`, and `onStatus`. + +```go +type orchestrator struct { + client llmCaller +} + +type pipelineState struct { + client llmCaller + repoRoot string + cfg *config.Config + cache *MetadataCache + affectedDirs map[string]bool + precalculatedHashes map[string]string + onStatus StatusFunc +} +``` + +`Run` is the entry point. It acquires a lock via `security.AcquireLock`, loads `.gitignore`, loads the metadata cache, and dispatches to `RunInit` or `RunUpdate`. `RunInit` builds the tree, hashes all code files, synthesizes the root summary, generates `architecture.md` and `quickstart.md`, updates `AGENTS.md`, and persists the cache. `RunUpdate` detects file changes, prunes stale cache entries, and regenerates only affected modules. + +`setupPipeline` initializes the cache and affected-dirs map. `teardownPipeline` cleans up stale module files and empty directories. `hashCodeFiles` computes SHA-256 hashes for a file list. `collectTreeDirs` returns the set of directories present in the tree. `updateAgentGuidelines` appends documentation references to `AGENTS.md`. + +### Hierarchical Synthesis + +#### Synthesis Pipeline (`synthesize.go`) + +`extractFileFacts` invokes the LLM for each extraction step per chunk of a file. It strips outer markdown fences, filters "none" facts, and caches results keyed by file SHA-256. Read failures are swallowed via `onStatus`; LLM and chunking failures propagate. + +`synthesizeChildren` processes child directories recursively, synthesizing subsystem summaries before individual files. `collectComponents` combines subsystem summaries and file facts into a unified component list. `synthesizeNode` reduces all components for a directory into a final summary, writes it to `modules/.md`, and persists the cache. If no components are produced, the cached summary is deleted and the markdown file is removed. + +```go +func extractFileFacts(ctx context.Context, p *pipelineState, f string, siblingFiles []string) (string, error) +func synthesizeChildren(ctx context.Context, p *pipelineState, node *DirNode) ([]string, error) +func collectComponents(ctx context.Context, p *pipelineState, node *DirNode, subsystems []string) ([]string, error) +func synthesizeNode(ctx context.Context, p *pipelineState, node *DirNode) (string, error) +``` + +`buildExtractionPrompt` constructs the user prompt for a single extraction step. `maxExtractionOverhead` computes the maximum characters available for extraction given system prompt and file context. `toSafeMarkdownFilename` sanitizes a module path for use as a filename. + +#### Text Reduction & Chunking (`chunking.go`) + +`reduceWithLLM` calls the LLM to reduce a batch of items. `reduceInChunks` splits oversized items with an 800-character overlap, batches remaining items, and reduces each batch. If total output reaches 95% of input, reduction stops and results are concatenated. Otherwise, the intermediate results are recursively reduced. + +`chunkTextWithOverlap` splits text with overlap validation. `stripOuterMarkdownFence` removes surrounding code fences. `calculateFileLimit` computes the maximum processing characters from context window minus system prompt overhead. `isNoneFact` filters empty, "none", and "n/a" facts. `expandOversizedItems` splits items exceeding the limit. `batchItems` groups remaining items. `countRunes` counts Unicode code points. `reduceItems` orchestrates the reduction with a custom function. + +### Error Handling Summary + +| Operation | Failure Mode | Handling | +|-----------|-------------|----------| +| Lock acquisition | `security.AcquireLock` fails | Fatal error returned | +| Gitignore load | `tools.LoadGitignore` fails | Fatal error returned | +| Code discovery | `tools.DiscoverCodeFiles` fails | Fatal error returned | +| File read (extraction) | `tools.ReadFileSafely` fails | Swallowed, logged via `onStatus` | +| LLM call | `CallLLM` fails | Propagated as wrapped error | +| Chunking | Invalid `maxRunes` or `overlapRunes` | Returned as error | +| Disk write | `tools.WriteFileSafely` fails | Fatal error returned | +| Stale cleanup | `os.Remove` fails | Silently ignored | +| Cache save | Marshaling or write fails | Fatal error returned | +| HTTP non-200 | Response body read | Wrapped with status code | +| Context cancellation | Any operation | Propagated as error | --- -## Subsystem: `internal/engine` - -### Module Responsibility and Data Flow -The `internal/engine` module is an orchestration engine using a hierarchical Map-Reduce pattern to transform source code into structured documentation via LLMs. It supports `ModeInit` (initial generation) and `ModeUpdate` (incremental updates). - -**Data Flow Lifecycle:** -1. **Initialization**: The `Runner` acquires an exclusive repository lock via `security.AcquireLock`. -2. **Discovery**: The `Tree` logic converts file paths into a hierarchical `DirNode` structure. -3. **Change Detection**: The `Orchestrator` compares SHA256 hashes against `MetadataCache`. Changes propagate upward through the `DirNode` tree to mark nodes as "affected." -4. **Hierarchical Synthesis**: - * **Leaf Processing**: Files undergo `Chunking` (recursive reduction if context limits are exceeded) followed by LLM extraction. - * **Aggregation**: Results aggregate from file $\rightarrow$ module $\rightarrow$ directory levels. -5. **Persistence**: Writes synthesized Markdown and the `MetadataCache` (JSON) to the `docsDir`. - -### Core Components - -#### Orchestration and Execution -* **Runner (`runner.go`)**: Entry point; manages execution lifecycle and repository locking. -* **Orchestrator (`orchestrator.go`)**: Manages cache invalidation, directory pruning, and global (root-level) documentation regeneration. -* **Synthesizer (`synthesize.go`)**: Traverses the `DirNode` tree; decides between cached summaries or new LLM extractions based on "affected" status. - -#### Data Processing -* **Chunking and Reduction (`chunking.go`)**: Implements recursive reduction for oversized items using expansion (overlapping segments), batching, and consolidation. Recursion terminates if output is $\ge 95\%$ of input. -* **Markdown Processing (`markdown.go`)**: Strips triple-backtick or JSON fences from LLM output to extract raw content. -* **Metadata Cache (`cache.go`)**: Persists SHA256 fingerprints. Only version `1` is supported; version mismatches trigger a clean state. -* **Tree and Change Analysis (`tree.go`)**: Tracks file status (`StatusAdded`, `StatusModified`, `StatusDeleted`) and propagates impact up the directory hierarchy. - -#### Infrastructure -* **LLM Client (`client.go`)**: Synchronous HTTP client for the `api/chat` protocol. Performs no retry logic or circuit breaking. -* **Constants (`constants.go`)**: - * `defaultHTTPTimeout`: 10 minutes. - * `contextWindowAllocRatio`: 0.75 (75% for content, 25% for overhead). - * `defaultChunkOverlap`: 800 tokens. - * `maxErrorBodyBytes`: 1 KB. -* **Utilities (`utils.go \ No newline at end of file +## Security Subsystem (`security.go`) + +### Module Responsibility + +This module provides two non-negotiable security primitives for a repository-level tool: + +1. **Path traversal prevention** — `SafeResolve` ensures any input path resolves strictly inside the repository root, even when symlinks are involved. +2. **Process-level locking** — `AcquireLock` / `SimpleLock` / `Unlock` prevents concurrent execution by multiple processes of the same tool. + +**Data flow:** `SafeResolve(repoRoot, inputPath)` → `(string, error)`. `AcquireLock(repoRoot)` → `(*SimpleLock, error)`. `(*SimpleLock).Unlock()` → `error`. No global mutable state exists; all constants and sentinel errors are immutable. + +### Path Traversal Prevention + +#### `SafeResolve(repoRoot, inputPath string) (string, error)` + +Cleans `inputPath` and verifies the resolved result lies strictly inside the repository root. + +**Algorithm:** + +1. `filepath.Abs(repoRoot)` — canonicalize the repository root to an absolute path. +2. `filepath.EvalSymlinks(repoRoot)` — resolve symlinks on the repository root itself. +3. Walk up the input path from the root, stopping at the first physically existing ancestor (checked via `os.Lstat`). +4. `filepath.EvalSymlinks` on that existing ancestor. +5. Rebuild the full path by appending the non-existent suffix components. +6. `filepath.Rel` computes the relative path from the resolved root to the rebuilt path. If the result is not `""`, return `ErrPathTraversal`. + +**Error propagation:** Returns `(string, error)`. Errors are wrapped with `fmt.Errorf` using `%w` for chainability. `ErrPathTraversal` is returned directly as a sentinel. + +### Process-Level Locking + +#### `AcquireLock(repoRoot string) (*SimpleLock, error)` + +Acquires a simple file lock in `repoRoot` using `O_EXCL` for atomicity. + +**Algorithm:** + +1. Resolve the lock file path using `SafeResolve`. +2. `os.OpenFile` with `O_WRONLY|O_CREATE|O_EXCL`. If the file already exists, return `ErrLockHeld`. +3. Write the current process PID (`os.Getpid`) to the file. +4. Return a `*SimpleLock` handle. + +**Error propagation:** If `OpenFile` fails with `os.IsExist`, wraps `ErrLockHeld`. If `Write` fails, the file is closed and removed before returning a wrapped error. + +#### `(*SimpleLock) Unlock() error` + +Releases the lock by closing the file and removing it. Idempotent and thread-safe. + +**Algorithm:** + +1. Acquire the internal `sync.Mutex` (`l.mu`). +2. If `l.closed` is true, return `nil`. +3. Close the file descriptor. +4. Remove the lock file. If both close and remove fail, return the error from the non-nil one. + +**Error propagation:** If close succeeds but remove fails (and is not `os.IsNotExist`), the remove error is assigned to `err`. If close fails, the remove error is ignored. + +### Types + +#### `SimpleLock` + +```go +type SimpleLock struct { + lockPath string + file *os.File + mu sync.Mutex + closed bool +} +``` + +- `lockPath` — absolute path to the lock file. +- `file` — open file descriptor for the lock. +- `mu` — protects `closed` field. +- `closed` — set to `true` in `Unlock()`; guards idempotency. + +### Constants and Errors + +**Constants** + +- `LockFileName` = `".code-reducer.lock"` +- `defaultFilePerm` = `0644` + +**Error Variables** + +- `ErrPathTraversal` = `errors.New("security violation: path traversal detected")` +- `ErrLockHeld` = `errors.New("lock is already held by another process")` + +### External I/O Interactions + +All operations target the local filesystem. No network, database, or API interactions. + +| Operation | Purpose | +|-----------|---------| +| `filepath.Abs` | Resolves absolute path for `repoRoot` | +| `filepath.EvalSymlinks` | Resolves symlinks on `repoRoot` and intermediate ancestors | +| `os.Lstat` | Checks existence of path components during traversal resolution | +| `os.OpenFile` | Opens the lock file with `O_WRONLY|O_CREATE|O_EXCL` | +| `os.Remove` | Removes the lock file during `Unlock()` and on write failure in `AcquireLock()` | +| `os.Getpid` | Reads process ID (no external I/O) | +| `filepath.Rel` | Computes relative path to verify containment | + +### Synchronization + +- `SimpleLock.Unlock()` uses `sync.Mutex` (`l.mu`) to protect the `closed` field. +- `AcquireLock()` is a package-level function that creates a new `SimpleLock` instance; it does not use any synchronization mechanism. + +### Error Handling Summary + +- Errors are not logged or printed; they propagate to the caller. +- No `panic`/`recover` is used. +- Sentinel errors (`ErrPathTraversal`, `ErrLockHeld`) are returned directly. +- All other errors are wrapped with `fmt.Errorf` using `%w` for chainability. + +--- + +## Tools Subsystem (`internal/tools`) + +### Module Responsibility + +This module provides safe filesystem operations and repository verification utilities for source code management workflows. It operates on a repository root (`repoRoot`) and resolves virtual paths through a security layer to prevent path traversal attacks. All operations are disk I/O only—no network, database, or external API calls. + +**Data Flow:** + +``` +caller → virtualPath → security.SafeResolve → filesystem path → os.* operations +``` + +All public functions accept a `repoRoot` parameter and resolve virtual paths before accessing the filesystem. Errors are propagated as return values; no panics occur. + +### File Tools (`file_tools.go`) + +#### Safe File Access + +**`FileExistsSafely(repoRoot string, virtualPath string) bool`** + +Checks whether a file exists at the resolved filesystem path. Swallows `os.Stat` errors and returns `false` on any failure. + +**`ReadFileSafely(repoRoot string, virtualPath string) ([]byte, error)`** + +Reads file contents from the resolved filesystem path. Returns the file data and an error. Swallows `os.Open` errors. + +**`RemoveFileSafely(repoRoot string, virtualPath string) error`** + +Removes a file at the resolved filesystem path. Ignores `os.IsNotExist` errors (file may already be absent). Propagates all other errors. + +#### Atomic File Writes + +**`WriteFileAtomic(targetPath string, data []byte, perm os.FileMode) error`** + +Writes data to `targetPath` atomically: writes to a temporary file first, then renames to the target path. Wraps all sub-errors with `fmt.Errorf("failed to %s: %w", ...)`. + +**`WriteFileSafely(repoRoot string, virtualPath string, content []byte) error`** + +Writes content to the resolved filesystem path. Uses atomic write semantics. + +#### Gitignore Integration + +**`EnsureGitignoreHasLockfile(repoRoot string) error`** + +Ensures `.code-reducer.lock` is included in `.gitignore` to prevent it from being committed. + +**`LoadGitignore(repoRoot string) ([]string, error)`** + +Loads `.gitignore` patterns from the repository root. Returns the parsed patterns and an error. + +**`ShouldIgnoreFile(relPath string, gitIgnore *ignore.GitIgnore) bool`** + +Determines whether a relative path matches any loaded gitignore patterns. Uses `ignore.CompileIgnoreLines` for pattern parsing (in-memory, no external I/O). + +### Global State + +| Variable | Type | Mutability | +|---|---|---| +| `skippedFileNames` | `map[string]bool` | Immutable after init | +| `skippedExtensions` | `map[string]bool` | Immutable after init | +| `skippedTestSuffixes` | `[]string` | Immutable after init | + +These maps are initialized once and never modified. They define the multi-layer file filtering rules applied during file discovery. + +### Git Tools (`git_tools.go`) + +#### Repository Verification + +**`VerifyGitRepo(repoRoot string) error`** + +Verifies whether `repoRoot` resides inside a git repository by executing `git rev-parse --is-inside-work-tree` in the specified directory. + +**Process Execution Details:** + +- Spawns `git` as a child process via `exec.Command("git", "--no-pager", "rev-parse", "--is-inside-work-tree")` +- Redirects child process stderr to a `bytes.Buffer` for capture +- Sets child process working directory to `repoRoot` + +**Error Handling:** + +- If `cmd.Run()` returns an error, the function wraps it with `fmt.Errorf` and returns the wrapped error (including captured stderr content) +- If `cmd.Run()` returns nil, returns nil (success) +- No panics or swallowed errors \ No newline at end of file diff --git a/wiki/modules/internal/config/README.md b/wiki/modules/internal/config/README.md index 67b2ea7..a03dab8 100644 --- a/wiki/modules/internal/config/README.md +++ b/wiki/modules/internal/config/README.md @@ -1,86 +1,136 @@ -# Package `internal/config` +# internal/config ## Module Responsibility -The `internal/config` package manages the lifecycle, hierarchical resolution, and persistence of configuration settings for the Code-Reducer system. It provides the mechanism to merge settings from multiple sources—including command-line flags, environment variables, and configuration files—into a unified `Config` struct. This configuration directs an LLM-based pipeline through specific extraction and synthesis phases. -### Data Flow -1. **Ingestion**: The system gathers data from the local filesystem (via `.code-reducer.yaml`), environment variables, and CLI arguments. -2. **Resolution**: A hierarchical merge process applies overrides based on a defined priority: CLI flags > Environment variables > Configuration file values > System defaults. -3. **Validation**: Numerical parameters, such as context size, undergo validation to ensure they meet non-zero requirements. -4. **Output**: A validated `*Config` pointer is returned for use in LLM prompt construction and execution logic. -5. **Persistence**: Configuration changes are serialized to YAML and written to the filesystem using an atomic rename pattern to ensure integrity. +The `internal/config` package manages the configuration lifecycle for Code-Reducer, an AI-driven code analysis and documentation generation tool. It handles three concerns: (1) persistence of a `.code-reducer.yaml` configuration file, (2) resolution of configuration values from multiple sources with defined priority ordering, and (3) application of system defaults to empty or zero-valued fields. -## Data Structures +**Data flow:** `ResolveConfig` is the primary entry point. It loads a base configuration from disk via `loadBaseConfig` (which delegates to `LoadConfig` in `io.go`), applies system defaults via `ApplyDefaults`, then resolves individual fields (`ModelID`, `OllamaBaseURL`, `OllamaNumCtx`) by merging CLI flags, environment variables, and config file values. The `Ignore` slice is deduplicated via `deduplicate`. The resolved `Config` struct is returned to callers. -### `Config` -The primary schema representing the system's operational parameters. +--- -| Field | Type | Description | +## Configuration Persistence (`io.go`) + +### Public API + +| Function | Parameters | Return Types | +|---|---|---| +| `ConfigExists` | `cwd string` | `bool` | +| `LoadConfig` | `cwd string` | `*Config`, `error` | +| `SaveConfig` | `cwd string`, `cfg *Config` | `error` | + +### Internal Functions + +| Function | Parameters | Return Types | +|---|---|---| +| `getConfigPath` | `cwd string` | `string` | +| `formatYAML` | `data []byte` | `string` | + +### File System Interactions + +- **Disk reads:** `os.Stat` in `ConfigExists`; `os.ReadFile` in `LoadConfig`. +- **Disk writes:** `tools.WriteFileAtomic` in `SaveConfig`. +- No network, database, or API interactions. + +### Error Propagation + +- `LoadConfig` returns `(*Config, error)`. If `os.ReadFile` fails, the raw error is returned. If YAML parsing fails, the error is wrapped with `fmt.Errorf("failed to parse yaml config: %w", err)`. +- `SaveConfig` returns `error`. If `yaml.Marshal` fails, the error is wrapped with `fmt.Errorf("failed to marshal yaml: %w", err)` before being passed to `tools.WriteFileAtomic`. +- `ConfigExists` derives a boolean from `os.Stat` error check (`err == nil`). +- `getConfigPath` and `formatYAML` have no error handling; failures result in panics. + +--- + +## Configuration Resolution (`resolve.go`) + +### Public API + +| Function | Parameters | Return Type | |---|---|---| -| `ModelID` | `string` | The identifier for the LLM model. | -| `OllamaBaseURL` | `string` | The base endpoint for the local Ollama API. | -| `OllamaNumCtx` | `int` | The context window size for the LLM. | -| `DocsDir` | `string` | The target directory for generated documentation. | -| `SystemPrompt` | `string` | The primary system message for the LLM. | -| `ModuleSynthesisPrompt` | `string` | Prompt template for module-level synthesis. | -| `ArchitecturePrompt` | `string` | Prompt template for architectural analysis. | -| `FileFactConsolidationPrompt` | `string` | Prompt template for consolidating file facts. | -| `ExtractionSteps` | `[]ExtractionStep` | A sequence of extraction phases for the analysis pipeline. | -| `Ignore` | `[]string` | A list of paths or patterns to be excluded from analysis. | - -### `ExtractionStep` -Defines a single phase in the multi-step LLM extraction process. - -| Field | Type | Description | +| `ApplyDefaults` | `cfg *Config` | (none) | +| `ResolveConfig` | `repoRoot string`, `modelIDFlag string`, `numCtxFlag string` | `(*Config, error)` | + +### Internal Functions + +| Function | Parameters | Return Type | |---|---|---| -| `Name` | `string` | The identifier for the extraction phase. | -| `Prompt` | `string` | The specific prompt utilized for this phase. | +| `deduplicate` | `a []string` | `[]string` | +| `loadBaseConfig` | `repoRoot string` | `(*Config, error)` | +| `resolveModelID` | `cfgVal string`, `flagVal string` | `string` | +| `resolveBaseURL` | `cfgVal string` | `string` | +| `parsePositiveInt` | `val string`, `errMsg string` | `(int, error)` | +| `resolveNumCtx` | `cfgVal int`, `flagVal string` | `(int, error)` | + +### External I/O Interactions -## Hierarchical Resolution +- **Disk I/O:** `loadBaseConfig` calls `LoadConfig(repoRoot)` from `io.go`. If the file does not exist (`os.ErrNotExist`), an empty `Config{}` is returned instead of an error. Other read errors are wrapped and returned. +- **Environment Variables:** Three environment variables are read via `os.Getenv`: + - `CodeReducerModelIDEnvKey` in `resolveModelID` + - `OllamaBaseURLEnvKey` in `resolveBaseURL` + - `OllamaNumCtxEnvKey` in `resolveNumCtx` +- No network, database, or API calls. -The `ResolveConfig` function implements the precedence logic to determine the final state of a `Config` instance. +### Error Propagation -### Resolution Precedence -For each parameter, the first non-default/non-empty value found in the following order is selected: -1. **Command-line Flags** (e.g., `modelIDFlag`, `numCtxFlag`). -2. **Environment Variables** (defined by `CodeReducerModelIDEnvKey`, `OllamaNumCtxEnvKey`, and `OllamaBaseURLEnvKey`). -3. **Configuration File** (values stored in `.code-reducer.yaml`). -4. **System Defaults** (hardcoded values). +- `ResolveConfig` returns errors from `loadBaseConfig` (which wraps non-`os.ErrNotExist` errors from `LoadConfig`) and from `resolveNumCtx` (which wraps errors from `parsePositiveInt`). +- `ApplyDefaults` and `deduplicate` swallow all errors silently. +- `resolveModelID` and `resolveBaseURL` return values without errors; callers handle the empty-string case implicitly. +- `parsePositiveInt` returns a formatted error string via `fmt.Errorf`. +- Errors are propagated via Go's `error` return convention; no panics or exceptions are used. -### Numerical Validation -The resolution of `OllamaNumCtx` requires that the value—whether sourced from a flag, environment variable, or file—is a valid integer and greater than zero. If an environment variable is non-empty but fails `strconv.Atoi` or is $\le 0$, `ResolveConfig` returns an error. +--- + +## Configuration Schema (`config.go`) + +### Constants + +| Constant | Value | +|---|---| +| `ConfigFileName` | `.code-reducer.yaml` | +| `ConfigFileName` | `ConfigFileName` (referenced in `io.go` and `resolve.go`) | +| `configFilePerm` | `0600` | +| `configFilePerm` | `configFilePerm` (referenced in `io.go`) | +| `OllamaDefaultBaseURL` | `http://localhost:11434` | +| `OllamaDefaultModelID` | `ornith:9b` | +| `OllamaDefaultNumCtx` | `8192` | +| `DefaultDocsDir` | `wiki` | +| `CodeReducerModelIDEnvKey` | `CODE_REDUCER_MODEL_ID` | +| `OllamaBaseURLEnvKey` | `OLLAMA_BASE_URL` | +| `OllamaNumCtxEnvKey` | `OLLAMA_NUM_CTX` | + +### Default Prompts + +The following multi-line string constants define system prompts for downstream synthesis steps: + +- `DefaultSystemPrompt` — Code-Reducer role and rules (extract facts, no speculation, no code blocks) +- `DefaultModuleSynthesisPrompt` — Folder summary generation +- `DefaultArchitecturePrompt` — Global architecture document generation +- `DefaultQuickstartPrompt` — Developer quickstart guide generation +- `DefaultFileFactConsolidationPrompt` — Fact consolidation and deduplication + +### Structs + +| Type | Fields | +|---|---| +| `ExtractionStep` | `Name` string, `Prompt` string | +| `Config` | `ModelID` string, `OllamaBaseURL` string, `OllamaNumCtx` int, `DocsDir` string, `SystemPrompt` string, `ModuleSynthesisPrompt` string, `ArchitecturePrompt` string, `QuickstartPrompt` string, `FileFactConsolidationPrompt` string, `ExtractionSteps` []ExtractionStep, `Ignore` []string | -### List and Default Initialization -* **Ignore List**: The `Ignore` slice is deduplicated during resolution. -* **Extraction Steps**: If the configuration file provides an empty list for `ExtractionSteps`, the system defaults to the `DefaultExtractionSteps` slice. +### Default Extraction Steps -## I/O and Persistence +`DefaultExtractionSteps` is a `[]ExtractionStep` containing: `API_SIGNATURES`, `BUSINESS_LOGIC`, `STATE_AND_CONCURRENCY`, `ERRORS_AND_SIDE_EFFECTS`. -### Filesystem Operations -The module utilizes a `ConfigFileName` (`.code-reducer.yaml`) within a specified directory to manage persistence. +--- -* **`ConfigExists`**: Checks for the existence of the configuration file using `os.Stat`. All errors encountered during this operation are swallowed, and the function returns `false`. -* **`LoadConfig`**: Reads the configuration file via `os.ReadFile` and unmarshals the content via `yaml.Unmarshal`. Errors from file reading or YAML parsing are wrapped and returned to the caller. -* **`SaveConfig`**: Implements an atomic write pattern to prevent file corruption: - 1. Serializes the `Config` struct to YAML bytes via `yaml.Marshal`. - 2. Creates a temporary file in the target directory using `os.CreateTemp`. - 3. Writes the data and ensures durability via `os.Sync`. - 4. Sets file permissions via `os.Chmod`. - 5. Closes the file and uses `os.Rename` to atomically replace the target file. - * **Note**: The deferred cleanup (via `os.Remove`) in `SaveConfig` does not contain `recover()`; panics during the deferred execution are not handled. +## Resolution Priority Ordering -## Constants and Defaults +The following priority ordering is enforced by `ResolveConfig` and its helpers: -### Environment Variable Keys -| Key | Purpose | +| Field | Priority Order | |---|---| -| `CODE_REDUCER_MODEL_ID` | Overrides the LLM `ModelID`. | -| `OLLAMA_BASE_URL` | Overrides the `OllamaBaseURL`. | -| `OLLAMA_NUM_CTX` | Overrides the `OllamaNumCtx`. | - -### Default Extraction Pipeline (`DefaultExtractionSteps`) -When no custom extraction steps are provided, the system executes these four stages: -1. **`API_SIGNATURES`**: Extracts public types, functions, and methods. -2. **`BUSINESS_LOGIC`**: Synthesizes the primary domain problem and high-level algorithmic steps. -3. **`STATE_AND_CONCURRENCY`**: Identifies mutable state and synchronization mechanisms. Returns "No mutable state" if none are found. -4. **`ERRORS_AND_SIDE_EFFECTS`**: Details external I/O interactions and error propagation. Returns "No external side effects" if no I/O is detected. \ No newline at end of file +| `ModelID` | CLI flag → `CODE_REDUCER_MODEL_ID` env var → config file value | +| `OllamaBaseURL` | `OLLAMA_BASE_URL` env var → config file value | +| `OllamaNumCtx` | CLI flag → `OLLAMA_NUM_CTX` env var → config file value | +| `DocsDir` | System default (`wiki`) when empty/zero | +| `ExtractionSteps` | System default when empty | +| All prompt fields | System default when empty | + +`DocsDir` is additionally trimmed and path-cleaned during default application. \ No newline at end of file diff --git a/wiki/modules/internal/engine/README.md b/wiki/modules/internal/engine/README.md index 3d4afea..162cdaa 100644 --- a/wiki/modules/internal/engine/README.md +++ b/wiki/modules/internal/engine/README.md @@ -1,77 +1,132 @@ -# Internal Engine Architecture Documentation +# internal/engine — Codebase Documentation Synthesis Engine -## Module Responsibility and Data Flow +## Module Responsibility -The `internal/engine` module is an automated documentation orchestration engine that employs a hierarchical Map-Reduce pattern to transform repository source code into structured technical documentation using Large Language Models (LLMs). It facilitates both initial documentation generation (`ModeInit`) and incremental updates (`ModeUpdate`) by implementing a change-detection pipeline based on file integrity hashes and directory-level impact analysis. +The `internal/engine` package implements an LLM-driven pipeline that automatically generates and maintains codebase documentation (architecture overview, quickstart guide, module details, and agent guidelines). The engine operates in two modes: -### Data Flow Lifecycle -1. **Initialization & Locking**: The `Runner` acquires an exclusive repository lock and ensures the `.gitignore` contains the lockfile. -2. **Discovery & Tree Construction**: `Tree` logic converts a flat list of file paths into a hierarchical `DirNode` structure. -3. **Change Detection**: The `Orchestrator` compares current file SHA256 hashes against the `MetadataCache`. Changes are mapped to specific `FileChange` objects and propagated upward through the `DirNode` tree to identify "affected" directory nodes. -4. **Hierarchical Synthesis**: - * The `Synthesizer` performs a bottom-up traversal of the affected tree branches. - * **Leaf Processing**: Files are read, content is processed via `Chunking` (recursive reduction if context limits are exceeded), and LLM extraction is performed. - * **Aggregation**: Results from files and subdirectories are aggregated and passed to the LLM for higher-level synthesis. -5. **Persistence**: Synthesized Markdown documentation and the updated `MetadataCache` are written to the filesystem. +- **Init** — Performs a full synthesis of the entire codebase from scratch. +- **Update** — Detects file changes since the last run and regenerates only the affected documentation. + +Data flows from the repository root through a lock acquisition, gitignore parsing, code discovery, SHA-256 hashing, and tree construction. The orchestrator coordinates cache loading, change detection, hierarchical synthesis, and final persistence. --- -## Orchestration and Execution Control +## Core Infrastructure -### Runner (`runner.go`) -Serves as the primary entry point for the module. It manages the execution lifecycle, handles repository-level synchronization via `security.AcquireLock`, and delegates core logic to the `Orchestrator`. It supports two operational modes: `ModeInit` and `ModeUpdate`. +### LLM Client (`client.go`) -### Orchestrator (`orchestrator.go`) -Coordinates the end-to-end pipeline. It manages the transition from file discovery to hierarchical synthesis. It handles the logic for cache invalidation, directory pruning for deleted files, and determines whether global documentation (e.g., `architecture.md`) requires regeneration based on the impact of changes detected in the root directory. +The `llmCaller` interface abstracts LLM invocation. The unexported `llmClient` type implements it by serializing requests to the Ollama `/api/chat` endpoint and returning text responses. ---- +```go +type llmCaller interface { + CallLLM(ctx context.Context, systemPrompt, userPrompt string) (string, error) + NumCtx() int +} +``` + +`CallLLM` sends an HTTP POST with a 10-minute timeout. On a 200 OK, the full response body is read; on non-200, up to 1024 bytes of the error body are captured. Marshaling, request creation, and response parsing errors are wrapped with `fmt.Errorf`. The `onStatus` callback fires when `done_reason == "length"`. + +### Metadata Cache (`cache.go`) -## Hierarchical Synthesis and Data Processing +`MetadataCache` persists file state and configuration across engine runs. It stores a version, a `stepsHash` derived from model ID and all prompt templates, per-file SHA-256 hashes with associated facts, and module-to-path mappings. -### Synthesizer (`synthesize.go`) -Implements the bottom-up reduction algorithm. It traverses the `DirNode` tree, determining for each node whether to use cached summaries or perform new LLM-based extractions based on the "affected" status. It manages the recursive aggregation of file-level facts into module-level and directory-level summaries. +```go +type FileCacheEntry struct { + SHA256 string + Facts string +} -### Chunking and Reduction (`chunking.go`) -Addresses LLM context window constraints through a recursive reduction strategy: -* **Expansion**: Splits oversized items into overlapping segments. -* **Batching**: Groups items to optimize context window utilization. -* **Recursive Reduction**: Applies LLM-based consolidation to batches. -* **Convergence Check**: Stops recursion if a reduction step fails to decrease the input size by more than 5% (i.e., output $\ge$ 95% of input). +type MetadataCache struct { + Version int + StepsHash string + Files map[string]FileCacheEntry + Modules map[string]string +} +``` -### Markdown Processing (`markdown.go`) -Provides utilities for cleaning LLM output, specifically stripping surrounding triple-backtick Markdown or JSON fences to extract raw content. +`loadMetadataCache` reads `.metadata.json` from `docsDir`. If absent, it returns an empty cache; if the version is incompatible, it resets. `saveMetadataCache` writes indented JSON. `IsInitialized` checks file existence. `computeSHA256` hashes arbitrary file paths. --- -## State, Change Detection, and Cache Management +## Tree Construction & Change Detection -### Metadata Cache (`cache.go`) -Provides a persistence layer for tracking extracted documentation facts and file integrity. -* **Versioning**: Only version `1` is accepted; version mismatches trigger a clean cache state. -* **Integrity**: Uses SHA256 fingerprints of raw file content at the virtual path relative to the repository root. -* **Persistence**: Serializes state to a JSON file in the specified `docsDir`. +### Directory Tree (`tree.go`) + +`DirNode` represents a directory containing files and child directories. `buildTree` parses a file slice into a hierarchical `*DirNode` structure. + +```go +type DirNode struct { + Path string + Files []string + Children map[string]*DirNode +} +``` + +`FileChange` tracks a file's status (`StatusAdded`, `StatusModified`, `StatusDeleted`) and path. `propagateAffected` walks the tree upward, marking ancestors affected when a descendant is affected. `isNodeAffected` checks whether a node's files appear in the changed set or its module is missing from cache. `determineAffected` combines changed files with cache state to produce a final affected-dirs map. + +### Orchestrator (`orchestrator.go`) + +The `orchestrator` struct holds the `llmCaller` and coordinates the full pipeline. `pipelineState` is created fresh per invocation and contains `repoRoot`, `cfg`, `cache`, `affectedDirs`, `precalculatedHashes`, and `onStatus`. -### Tree and Change Analysis (`tree.go`) -Implements dependency impact analysis within the directory hierarchy. -* **Status Tracking**: Tracks files as `StatusAdded`, `StatusModified`, or `StatusDeleted`. -* **Upward Propagation**: If a child node (file or directory) is marked as affected, its status is recursively propagated to all ancestor `DirNode` objects. -* **Impact Detection**: Identifies affected nodes based on missing cache entries, modified hashes, or missing documentation files. +```go +type orchestrator struct { + client llmCaller +} + +type pipelineState struct { + client llmCaller + repoRoot string + cfg *config.Config + cache *MetadataCache + affectedDirs map[string]bool + precalculatedHashes map[string]string + onStatus StatusFunc +} +``` + +`Run` is the entry point. It acquires a lock via `security.AcquireLock`, loads `.gitignore`, loads the metadata cache, and dispatches to `RunInit` or `RunUpdate`. `RunInit` builds the tree, hashes all code files, synthesizes the root summary, generates `architecture.md` and `quickstart.md`, updates `AGENTS.md`, and persists the cache. `RunUpdate` detects file changes, prunes stale cache entries, and regenerates only affected modules. + +`setupPipeline` initializes the cache and affected-dirs map. `teardownPipeline` cleans up stale module files and empty directories. `hashCodeFiles` computes SHA-256 hashes for a file list. `collectTreeDirs` returns the set of directories present in the tree. `updateAgentGuidelines` appends documentation references to `AGENTS.md`. --- -## Infrastructure and Communication +## Hierarchical Synthesis -### LLM Client (`client.go`) -An HTTP client targeting the Ollama `api/chat` protocol. -* **Interface**: Abstraction via `llmCaller` allows for decoupling the engine from the specific transport. -* **Behavior**: Executes synchronous POST requests. It performs no retry logic on transport-level failures and contains no circuit-breaking mechanisms. - -### Configuration and Constants (`constants.go`) -Defines operational parameters, including: -* **Timeouts**: `defaultHTTPTimeout` (10 minutes). -* **Context Management**: A `contextWindowAllocRatio` of 0.75 reserves 75% of the context window for primary content, leaving 25% for metadata/overhead. -* **Chunking**: `defaultChunkOverlap` (800 tokens). -* **Limits**: `maxErrorBodyBytes` (1 KB) caps error response parsing. - -### Utilities (`utils.go`) -Provides filesystem path sanitization for generating `README.md` files and a nil-safe event logging adapter. \ No newline at end of file +### Synthesis Pipeline (`synthesize.go`) + +`extractFileFacts` invokes the LLM for each extraction step per chunk of a file. It strips outer markdown fences, filters "none" facts, and caches results keyed by file SHA-256. Read failures are swallowed via `onStatus`; LLM and chunking failures propagate. + +`synthesizeChildren` processes child directories recursively, synthesizing subsystem summaries before individual files. `collectComponents` combines subsystem summaries and file facts into a unified component list. `synthesizeNode` reduces all components for a directory into a final summary, writes it to `modules/.md`, and persists the cache. If no components are produced, the cached summary is deleted and the markdown file is removed. + +```go +func extractFileFacts(ctx context.Context, p *pipelineState, f string, siblingFiles []string) (string, error) +func synthesizeChildren(ctx context.Context, p *pipelineState, node *DirNode) ([]string, error) +func collectComponents(ctx context.Context, p *pipelineState, node *DirNode, subsystems []string) ([]string, error) +func synthesizeNode(ctx context.Context, p *pipelineState, node *DirNode) (string, error) +``` + +`buildExtractionPrompt` constructs the user prompt for a single extraction step. `maxExtractionOverhead` computes the maximum characters available for extraction given system prompt and file context. `toSafeMarkdownFilename` sanitizes a module path for use as a filename. + +### Text Reduction & Chunking (`chunking.go`) + +`reduceWithLLM` calls the LLM to reduce a batch of items. `reduceInChunks` splits oversized items with an 800-character overlap, batches remaining items, and reduces each batch. If total output reaches 95% of input, reduction stops and results are concatenated. Otherwise, the intermediate results are recursively reduced. + +`chunkTextWithOverlap` splits text with overlap validation. `stripOuterMarkdownFence` removes surrounding code fences. `calculateFileLimit` computes the maximum processing characters from context window minus system prompt overhead. `isNoneFact` filters empty, "none", and "n/a" facts. `expandOversizedItems` splits items exceeding the limit. `batchItems` groups remaining items. `countRunes` counts Unicode code points. `reduceItems` orchestrates the reduction with a custom function. + +--- + +## Error Handling Summary + +| Operation | Failure Mode | Handling | +|-----------|-------------|----------| +| Lock acquisition | `security.AcquireLock` fails | Fatal error returned | +| Gitignore load | `tools.LoadGitignore` fails | Fatal error returned | +| Code discovery | `tools.DiscoverCodeFiles` fails | Fatal error returned | +| File read (extraction) | `tools.ReadFileSafely` fails | Swallowed, logged via `onStatus` | +| LLM call | `CallLLM` fails | Propagated as wrapped error | +| Chunking | Invalid `maxRunes` or `overlapRunes` | Returned as error | +| Disk write | `tools.WriteFileSafely` fails | Fatal error returned | +| Stale cleanup | `os.Remove` fails | Silently ignored | +| Cache save | Marshaling or write fails | Fatal error returned | +| HTTP non-200 | Response body read | Wrapped with status code | +| Context cancellation | Any operation | Propagated as error | \ No newline at end of file diff --git a/wiki/modules/internal/security/README.md b/wiki/modules/internal/security/README.md index d487f51..3d852b0 100644 --- a/wiki/modules/internal/security/README.md +++ b/wiki/modules/internal/security/README.md @@ -1,130 +1,125 @@ -# Module: `internal/security` — Path Traversal Prevention and Process-Level Locking +# security.go — Repository Security Module ## Module Responsibility -This module provides two security primitives scoped to a repository root: +This module provides two non-negotiable security primitives for a repository-level tool: -1. **Path traversal prevention** via `SafeResolve`, which validates that user-supplied paths resolve within the repository boundary, resolving symlinks on existing ancestor parts before comparison. -2. **Cross-process lock management** via `SimpleLock`/`AcquireLock`/`Unlock`, which ensures only one process holds exclusive access to a protected resource at any given time through an atomic file-based lock mechanism. +1. **Path traversal prevention** — `SafeResolve` ensures any input path resolves strictly inside the repository root, even when symlinks are involved. +2. **Process-level locking** — `AcquireLock` / `SimpleLock` / `Unlock` prevents concurrent execution by multiple processes of the same tool. -Both primitives operate against the local filesystem; no network, database, or remote API interactions occur within production code. All errors are returned to callers and propagated via Go's `error` interface; panics are not raised by this module. +**Data flow:** `SafeResolve(repoRoot, inputPath)` → `(string, error)`. `AcquireLock(repoRoot)` → `(*SimpleLock, error)`. `(*SimpleLock).Unlock()` → `error`. No global mutable state exists; all constants and sentinel errors are immutable. -## Data Flow Overview +--- -1. Callers invoke `SafeResolve(repoRoot, inputPath)` with a candidate path string and an anchor directory; the function returns either a resolved absolute path confined within `repoRoot` or an error (typically `ErrPathTraversal`). -2. Callers invoke `AcquireLock(repoRoot)` to obtain a lock handle; if contention is detected—either via OS-level atomic failure of `O_EXCL` or presence of a stale lockfile—the function returns an error wrapping the sentinel `ErrLockHeld`. -3. When release is needed, callers call `Unlock()` on the returned `*SimpleLock`; this method is idempotent and thread-safe with respect to itself, closing the file descriptor and removing the lockfile from disk under a mutex guard. +## Path Traversal Prevention -## Error Definitions — `errors.go` +### `SafeResolve(repoRoot, inputPath string) (string, error)` -The module declares two sentinel error variables used as context markers during error propagation: +Cleans `inputPath` and verifies the resolved result lies strictly inside the repository root. -| Sentinel | Declaration Scope | Usage Context | -|---|---|---| -| `ErrPathTraversal` | package-level in `internal/security/errors.go` | Returned by `SafeResolve` when the resolved path escapes outside `repoRoot`. The original traversal signal is swallowed and replaced with a formatted message carrying `inputPath` as context. | -| `ErrLockHeld` | package-level in `internal/security/errors.go` | Returned by `AcquireLock` when another process holds the lock or a stale lockfile persists; returned via `%w` wrapping of the underlying OS error with descriptive context. | +**Algorithm:** -No functions, imports beyond `errors`, or runtime operations exist in this file. Error propagation from these sentinels (e.g., further wrapping via `fmt.Errorf`) occurs elsewhere, not here. +1. `filepath.Abs(repoRoot)` — canonicalize the repository root to an absolute path. +2. `filepath.EvalSymlinks(repoRoot)` — resolve symlinks on the repository root itself. +3. Walk up the input path from the root, stopping at the first physically existing ancestor (checked via `os.Lstat`). +4. `filepath.EvalSymlinks` on that existing ancestor. +5. Rebuild the full path by appending the non-existent suffix components. +6. `filepath.Rel` computes the relative path from the resolved root to the rebuilt path. If the result is not `""`, return `ErrPathTraversal`. -## Path Resolution — `SafeResolve` +**Error propagation:** Returns `(string, error)`. Errors are wrapped with `fmt.Errorf` using `%w` for chainability. `ErrPathTraversal` is returned directly as a sentinel. -### Signature +--- -```go -func SafeResolve(repoRoot, inputPath string) (string, error) -``` +## Process-Level Locking + +### `AcquireLock(repoRoot string) (*SimpleLock, error)` -### Responsibility +Acquires a simple file lock in `repoRoot` using `O_EXCL` for atomicity. -Resolves a candidate path against an anchor root while preventing escape through symlinks and directory traversal. The function returns the cleaned absolute path if it remains strictly inside the resolved repository boundary; otherwise it returns `ErrPathTraversal`. +**Algorithm:** -### Algorithm Steps +1. Resolve the lock file path using `SafeResolve`. +2. `os.OpenFile` with `O_WRONLY|O_CREATE|O_EXCL`. If the file already exists, return `ErrLockHeld`. +3. Write the current process PID (`os.Getpid`) to the file. +4. Return a `*SimpleLock` handle. -1. Compute the absolute root directory from `repoRoot` via `filepath.Abs`, wrapping any resulting error with `%w`. -2. Resolve symlinks on the absolute root using `filepath.EvalSymlinks(absRoot)`, again wrapping errors with `%w`. -3. Walk up from the joined path (`absRoot + inputPath`) until a physically existing ancestor is found via repeated `os.Lstat(current)` calls; each Lstat failure that is not "not exist" is wrapped and returned immediately. -4. Resolve symlinks on the first physically existing ancestor via `filepath.EvalSymlinks(current)`, wrapping errors with `%w`. -5. Reconstruct the full target path from the resolved ancestor plus all previously-skipped components. -6. Verify that the reconstructed path remains inside the resolved root; reject if it escapes by returning a wrapped error using `ErrPathTraversal` with `inputPath` as context. +**Error propagation:** If `OpenFile` fails with `os.IsExist`, wraps `ErrLockHeld`. If `Write` fails, the file is closed and removed before returning a wrapped error. -### External I/O Operations +### `(*SimpleLock) Unlock() error` -| Operation | Function | Purpose | -|---|---|---| -| Absolute path resolution from relative input | `filepath.Abs(repoRoot)` | Establishes the canonical root baseline for comparison | -| Symlink resolution on root | `filepath.EvalSymlinks(absRoot)` | Prevents symlink-based escape at the boundary | -| Ancestor existence check | `os.Lstat(current)` loop | Walks up the path tree stopping at first physically existing parent | -| Symlink resolution on ancestor | `filepath.EvalSymlinks(current)` | Prevents symlink-based escape mid-path | +Releases the lock by closing the file and removing it. Idempotent and thread-safe. -## Lock Acquisition and Release — `SimpleLock` +**Algorithm:** -### Types +1. Acquire the internal `sync.Mutex` (`l.mu`). +2. If `l.closed` is true, return `nil`. +3. Close the file descriptor. +4. Remove the lock file. If both close and remove fail, return the error from the non-nil one. -#### `SimpleLock` Struct +**Error propagation:** If close succeeds but remove fails (and is not `os.IsNotExist`), the remove error is assigned to `err`. If close fails, the remove error is ignored. -| Field | Type | Mutability | Notes | -|---|---|---|---| -| `lockPath` | `string` | Modified once during acquisition, then read-only thereafter. Not mutated by any other method. | Absolute path of the lockfile within the repository root. | -| `file` | `*os.File` | Modified once during acquisition. Closed during unlock and removed from disk afterward. | File descriptor holding the exclusive lock; opened with `O_WRONLY\|O_CREATE\|O_EXCL`. | -| `mu` | `sync.Mutex` | Read via lock/unlock primitives only. Field itself is never reassigned after struct initialization. | Protects the close-and-remove sequence in `Unlock()`, making unlock atomic with respect to itself. | -| `closed` | `bool` | Set to `true` during unlock; read inside unlock for idempotency check (`if l.closed { return nil }`). | Tracks release state so subsequent calls return immediately without further I/O. | +--- -#### Methods +## Types -##### `AcquireLock(repoRoot string) (*SimpleLock, error)` +### `SimpleLock` + +```go +type SimpleLock struct { + lockPath string + file *os.File + mu sync.Mutex + closed bool +} +``` -**Responsibility**: Acquires an exclusive file-based lock within the provided repository root. Uses O_EXCL to ensure atomicity; failure implies another process holds the lock or a stale lockfile exists. +- `lockPath` — absolute path to the lock file. +- `file` — open file descriptor for the lock. +- `mu` — protects `closed` field. +- `closed` — set to `true` in `Unlock()`; guards idempotency. -**Algorithm Steps**: -1. Calls `SafeResolve(repoRoot, LockFileName)` to obtain the canonical lock file path inside the repo root. If this fails, the error propagates directly without wrapping. -2. Opens the lockfile with `os.OpenFile(lockPath, O_WRONLY\|O_CREATE\|O_EXCL, 0644)`. The OS-level atomicity of O_EXCL means failure indicates another writer holds the file or a stale lock persists; this is treated as an error condition requiring manual cleanup by the caller. -3. Writes the current process PID into the lockfile via `f.Write([]byte(fmt.Sprintf("%d\n", os.Getpid())))` for identification/inspection purposes. If this write fails, the method closes the file and removes it before returning a wrapped error. +--- -**Error Propagation**: -- Direct propagation when `SafeResolve` fails (no wrapping). -- Wraps with `%w` using `ErrLockHeld` plus context string describing stale lockfile when the OS reports `os.IsExist`. -- On write failure after successful OpenFile: closes file and removes it, then returns a wrapped error that swallows close/remove errors into a single formatted message—only the original write error is surfaced. +## Constants and Errors -##### `Unlock() error` (method on `*SimpleLock`) +### Constants -**Responsibility**: Releases the lock by closing the file descriptor and removing the lockfile from disk. Idempotent and thread-safe with respect to itself. +- `LockFileName` = `".code-reducer.lock"` +- `defaultFilePerm` = `0644` -**Algorithm Steps**: -1. Acquires `l.mu.Lock()` at entry; releases via `defer mu.Unlock()`. -2. Checks `if l.closed { return nil }`; if already closed, returns immediately without further I/O. -3. Closes the file descriptor (`l.file.Close()`). If this fails, the error is swallowed when reporting removal errors—only surfaces the remove error if it subsequently fails. -4. Attempts to remove the lockfile from disk via `os.Remove(l.lockPath)`. +### Error Variables -**Thread Safety**: The struct owns its own mutex; concurrent calls to `Unlock` on the same instance serialize through `mu.Lock()/defer mu.Unlock()`, making the operation atomic with respect to itself. Concurrent acquisition of different instances each creates independent `SimpleLock` objects with separate mutexes—no contention between instances is modeled or guaranteed by this code. +- `ErrPathTraversal` = `errors.New("security violation: path traversal detected")` +- `ErrLockHeld` = `errors.New("lock is already held by another process")` -## Test Coverage — `security_test.go` +--- -### Functions Under Test +## External I/O Interactions -| Function | Parameters | Return Values | -|---|---|---| -| `SafeResolve(repoRoot, inputPath)` | repository root path; input file/directory path to resolve | resolved absolute path (string); error | -| `AcquireLock(repoRoot)` | repository root path | lock object (`*SimpleLock`); error | +All operations target the local filesystem. No network, database, or API interactions. -### Test Fixture Operations +| Operation | Purpose | +|-----------|---------| +| `filepath.Abs` | Resolves absolute path for `repoRoot` | +| `filepath.EvalSymlinks` | Resolves symlinks on `repoRoot` and intermediate ancestors | +| `os.Lstat` | Checks existence of path components during traversal resolution | +| `os.OpenFile` | Opens the lock file with `O_WRONLY|O_CREATE|O_EXCL` | +| `os.Remove` | Removes the lock file during `Unlock()` and on write failure in `AcquireLock()` | +| `os.Getpid` | Reads process ID (no external I/O) | +| `filepath.Rel` | Computes relative path to verify containment | -All operations below target the local filesystem via Go's standard library: +--- -| Operation | Function | Purpose | Error Handling | -|---|---|---|---| -| Directory creation for test fixtures | `os.MkdirAll()` | Sets up `src/sub/` directories | Propagated to test framework via `t.Fatal(err)` — terminates immediately, no recovery. | -| File writing for traversal tests | `os.WriteFile()` | Creates `file.txt`, `..config` | Same as above; propagated to `t.Fatal(err)`. | -| Symlink creation (conditional) | `os.Symlink()` | Creates a symlink pointing outside the repo root | Swallowed and replaced with `t.Skip("Symlinks not supported on this OS/filesystem")`; test continues if operation fails. | +## Synchronization -### Test Assertions +- `SimpleLock.Unlock()` uses `sync.Mutex` (`l.mu`) to protect the `closed` field. +- `AcquireLock()` is a package-level function that creates a new `SimpleLock` instance; it does not use any synchronization mechanism. -- `security.SafeResolve(repoRoot, inputPath)` returns an error to its caller; tests verify both success and failure paths via a `wantErr` flag comparison (`t.Errorf(...)`). -- `security.AcquireLock(repoRoot)` returns `(lock, err)`. A second concurrent call failing with `AcquireLock() should have failed when lock is already held` confirms the production code signals contention via error return. +--- -## Concurrency and Synchronization Summary +## Error Handling Summary -- No package-level variables are declared anywhere in this module; all mutable state is encapsulated within individual `SimpleLock` instances. -- `Unlock()` self-synchronizes via `mu.Lock()/defer mu.Unlock()`, making it safe for concurrent calls on the same instance. -- Concurrent calls to `AcquireLock` on different instances each create their own independent `SimpleLock` with separate mutexes—no contention between instances is modeled or guaranteed by this code. -- The file-level lock (the `.code-reducer.lock` file itself) provides OS-level mutual exclusion, which complements the in-memory `mu`. -- No locks/mutexes are used within the test file itself; each `Test*` function runs independently in a single goroutine. \ No newline at end of file +- Errors are not logged or printed; they propagate to the caller. +- No `panic`/`recover` is used. +- Sentinel errors (`ErrPathTraversal`, `ErrLockHeld`) are returned directly. +- All other errors are wrapped with `fmt.Errorf` using `%w` for chainability. \ No newline at end of file diff --git a/wiki/modules/internal/tools/README.md b/wiki/modules/internal/tools/README.md index 09fb470..362e12d 100644 --- a/wiki/modules/internal/tools/README.md +++ b/wiki/modules/internal/tools/README.md @@ -1,153 +1,87 @@ -# Module: `internal/tools` — Repository Analysis Utilities +# Internal Tools Module -## Responsibility & Data Flow +## Module Responsibility -The `internal/tools` package provides two complementary capabilities for analyzing a code repository's structure: (1) safe file I/O operations with TOCTOU protection and atomic writes, and (2) Git command abstraction for process execution. All functions operate against a local filesystem only; no network calls, database interactions, or external API invocations occur. +This module provides safe filesystem operations and repository verification utilities for source code management workflows. It operates on a repository root (`repoRoot`) and resolves virtual paths through a security layer to prevent path traversal attacks. All operations are disk I/O only—no network, database, or external API calls. -The module's entry point is the `repoRoot` string passed to every public function. Data flows from caller → `internal/tools` package functions → OS subsystems (POSIX file system for I/O operations, git binary via subprocess execution). No mutable state exists within any function scope; all operations are stateless and idempotent with respect to repository content. +**Data Flow:** ---- - -## File Operations (`file_tools.go`) - -### Atomic Write — `WriteFileAtomic` - -```go -func WriteFileAtomic(targetPath string, data []byte, perm os.FileMode) error ``` - -Writes binary data to `targetPath` using a temp-file + rename pattern that prevents partial or corrupt state if interrupted. The directory is created with mode `0755`. Returns an error on any failure; returns `nil` only after the final rename succeeds. - -**Sequence:** -1. Create temporary file in same directory as target (ensures atomic rename semantics) -2. Write data to temp file, sync before close for crash safety -3. Close and flush via defer block with cleanup gated by success flag -4. Apply `perm` mode via `os.Chmod` -5. Atomic rename from temp path into final location - -Errors at any step are wrapped; deferred cleanup runs on failure paths to prevent partial temp file leakage. No panic/recover anywhere in this function. - -### Safe Read — `ReadFileSafely` - -```go -func ReadFileSafely(repoRoot string, virtualPath string) ([]byte, error) +caller → virtualPath → security.SafeResolve → filesystem path → os.* operations ``` -Resolves the virtual path against `repoRoot`, then reads via `os.ReadFile`. Wraps any read failure with context ("failed to read file content"). The path resolution error from `security.SafeResolve` propagates unwrapped. +All public functions accept a `repoRoot` parameter and resolve virtual paths before accessing the filesystem. Errors are propagated as return values; no panics occur. -### Safe Write — `WriteFileSafely` +--- -```go -func WriteFileSafely(repoRoot string, virtualPath string, content []byte) error -``` +## File Tools (`file_tools.go`) -Delegates to both `ReadFileSafely` (for path resolution) and `WriteFileAtomic`. Unwrapped `security.SafeResolve` error propagates through. Otherwise defers to `WriteFileAtomic`'s return value. +### Safe File Access -### Gitignore Integration (`file_tools.go`) +**`FileExistsSafely(repoRoot string, virtualPath string) bool`** -#### Lockfile Recording — `EnsureGitignoreHasLockfile` +Checks whether a file exists at the resolved filesystem path. Swallows `os.Stat` errors and returns `false` on any failure. -```go -func EnsureGitignoreHasLockfile(repoRoot string) error -``` +**`ReadFileSafely(repoRoot string, virtualPath string) ([]byte, error)`** -Ensures `.gitignore` contains a line matching `security.LockFileName`. Reads the existing file, appends the lock entry if absent (adding a newline before it when needed), and writes back atomically. Returns an error on read/write failures. Read errors from `.gitignore` (except `os.IsNotExist`) are wrapped with "error reading .gitignore". A non-existent `.gitignore` falls through to the write path, which will fail if the directory does not exist; no special handling for that intermediate state. +Reads file contents from the resolved filesystem path. Returns the file data and an error. Swallows `os.Open` errors. -#### Loading — `LoadGitignore` +**`RemoveFileSafely(repoRoot string, virtualPath string) error`** -```go -func LoadGitignore(repoRoot string) ([]string, error) -``` +Removes a file at the resolved filesystem path. Ignores `os.IsNotExist` errors (file may already be absent). Propagates all other errors. -Reads `.gitignore` from `repoRoot`, strips comments (`#`) and blank lines, returns remaining patterns as `[]string`. Returns `nil, nil` when no `.gitignore` exists (treated as success). Any other read failure propagates unwrapped. Scanner errors appended via `scanner.Err()` at end — if a scan error occurs mid-stream but lines have already been populated, the partial result still gets returned to the caller. +### Atomic File Writes -#### Path Filtering — `ShouldIgnoreFile` +**`WriteFileAtomic(targetPath string, data []byte, perm os.FileMode) error`** -```go -func ShouldIgnoreFile(relPath string, gitIgnore *ignore.GitIgnore) bool -``` +Writes data to `targetPath` atomically: writes to a temporary file first, then renames to the target path. Wraps all sub-errors with `fmt.Errorf("failed to %s: %w", ...)`. -Returns `true` when the given relative path should be ignored: matches via the provided GitIgnore object (if non-nil), contains a dot-prefixed component, or ends with `.egg-info`. No I/O; returns only boolean values. No error path. +**`WriteFileSafely(repoRoot string, virtualPath string, content []byte) error`** -#### Discovery — `DiscoverCodeFiles` +Writes content to the resolved filesystem path. Uses atomic write semantics. -```go -func DiscoverCodeFiles(repoRoot string, ignores []string) ([]string, error) -``` +### Gitignore Integration -Recursively walks `repoRoot` to collect relative file paths that pass ignore filtering. Skips directories whose names start with `.`, end with `.egg-info`, or match the compiled GitIgnore rules. Returns a list of valid relative paths and any walk error. Walk errors are wrapped with context (path + "error walking"). Relative-path computation failures are also wrapped. Directory entries matching ignore rules return `filepath.SkipDir`. Non-matching files append to the slice. +**`EnsureGitignoreHasLockfile(repoRoot string) error`** -**Walk sequence:** -1. Iterate via `filepath.WalkDir` from `repoRoot` -2. For each directory entry: skip if name starts with `.`, ends with `.egg-info`, or matches compiled GitIgnore patterns -3. For each file entry: run both ignore-checks (compiled rules + dot-prefixed / `.egg-info` suffix), collect those that pass +Ensures `.code-reducer.lock` is included in `.gitignore` to prevent it from being committed. ---- +**`LoadGitignore(repoRoot string) ([]string, error)`** -## Git Integration (`git_tools.go`) +Loads `.gitignore` patterns from the repository root. Returns the parsed patterns and an error. -### Command Execution — `RunGit` +**`ShouldIgnoreFile(relPath string, gitIgnore *ignore.GitIgnore) bool`** -```go -func RunGit(repoRoot string, args ...string) (string, error) -``` - -Executes a git command in the specified repo directory with `--no-pager`. Returns trimmed stdout and any error. - -**Sequence:** -1. Construct invocation: `git --no-pager ` in `repoRoot` -2. Spawn child process via `exec.Command("git", ...)` — subprocess runs in `repoRoot` -3. Capture stdout into `bytes.Buffer`, stderr separately buffered -4. If `cmd.Run()` succeeds: trim whitespace from captured stdout, return as string result -5. If `cmd.Run()` fails: trim whitespace from captured stderr, wrap with `fmt.Errorf("git command failed: %v, stderr: %s", err, trimmedErr)` - -The original error from `exec.Cmd` (e.g., `*exec.Error`, non-zero exit status) is embedded as `%v`; stderr text appended. No panic occurs on failure. +Determines whether a relative path matches any loaded gitignore patterns. Uses `ignore.CompileIgnoreLines` for pattern parsing (in-memory, no external I/O). -### Repository Validation — `VerifyGitRepo` +### Global State -```go -func VerifyGitRepo(repoRoot string) error -``` +| Variable | Type | Mutability | +|---|---|---| +| `skippedFileNames` | `map[string]bool` | Immutable after init | +| `skippedExtensions` | `map[string]bool` | Immutable after init | +| `skippedTestSuffixes` | `[]string` | Immutable after init | -Checks that `git` is available and that `repoRoot` is inside a git working tree. Invokes `git rev-parse --is-inside-work-tree`. If the returned error is non-nil, wraps with: `fmt.Errorf("not a git repository (or any of the parent directories): %w", err)`. Uses `%w` for wrapError-compatible wrapping (recoverable by unwrappers). +These maps are initialized once and never modified. They define the multi-layer file filtering rules applied during file discovery. --- -## Error Propagation Patterns +## Git Tools (`git_tools.go`) -### Disk I/O — File Operations +### Repository Verification -All functions interact exclusively with the local filesystem via standard OS calls (`os.ReadFile`, `os.CreateTemp`, `os.MkdirAll`, `os.Rename`, `os.Chmod`, `os.Open`). No network, database, or external API interactions exist. - -| Function | I/O Type | External System | -|---|---|---| -| `ReadFileSafely` | Disk read (`os.ReadFile`) | Local filesystem — target path resolved through `security.SafeResolve` before reading | -| `WriteFileAtomic` | Disk write (create temp → sync → chmod → rename) | Local filesystem — writes to a `.tmp.*` intermediate file then renames into place | -| `WriteFileSafely` | Disk read + disk write | Delegates path resolution and atomic write; no additional I/O beyond what the two sub-functions perform | -| `EnsureGitignoreHasLockfile` | Disk read (`os.ReadFile`) + disk write (`WriteFileAtomic`) | Reads `.gitignore`, appends lockfile pattern, writes back atomically | -| `LoadGitignore` | Disk read (`os.Open` + `bufio.Scanner`) | Reads `.gitignore`; returns `nil, nil` when file does not exist (no error) | -| `DiscoverCodeFiles` | Disk walk (`filepath.WalkDir`) | Recursively walks the repository root; skips directories matching ignore rules or dot-prefixed names | +**`VerifyGitRepo(repoRoot string) error`** -### Process I/O — Git Operations +Verifies whether `repoRoot` resides inside a git repository by executing `git rev-parse --is-inside-work-tree` in the specified directory. -The only external interaction is subprocess execution against the installed `git` binary. Stdout captured to `bytes.Buffer`, stderr separately buffered. `cmd.Run()` executes the command and returns the exit error. +**Process Execution Details:** -| Function | System | Mechanism | -|---|---|---| -| `RunGit` | External process (git) | `exec.Command("git", ...)` via `os/exec` — spawns child git subprocess in `repoRoot`. Stdout captured to `bytes.Buffer`, stderr separately buffered. `cmd.Run()` executes the command and returns the exit error. | -| `VerifyGitRepo` | External process (git) | `RunGit(repoRoot, "rev-parse", "--is-inside-work-tree")` — delegates to `RunGit` for execution; wraps non-nil return value with repository classification error | - ---- - -## Concurrency & State - -No mutable state exists within any function scope. All operations are stateless and idempotent with respect to repository content. No goroutine spawning or concurrent access patterns observed in this module. - ---- +- Spawns `git` as a child process via `exec.Command("git", "--no-pager", "rev-parse", "--is-inside-work-tree")` +- Redirects child process stderr to a `bytes.Buffer` for capture +- Sets child process working directory to `repoRoot` -## Notable Observations +**Error Handling:** -- **TOCTOU mitigation**: `WriteFileAtomic` uses a temp file + rename pattern, which avoids race conditions on the target path during writes -- **Sync before close**: `tmpFile.Sync()` is called before `Close()`, ensuring data reaches stable storage before the handle is released — relevant for crash safety but adds latency per write -- **Deferred cleanup in WriteFileAtomic**: On any error, `tmpFile.Close()` and `os.Remove(tmpName)` run via defer. This prevents partial temp file leakage on failure paths. The flag `success` gates whether to trigger cleanup -- **No panic/recover anywhere** — all errors are returned as typed values. No `panic`, no bare recovery blocks observed in this module or its tests \ No newline at end of file +- If `cmd.Run()` returns an error, the function wraps it with `fmt.Errorf` and returns the wrapped error (including captured stderr content) +- If `cmd.Run()` returns nil, returns nil (success) +- No panics or swallowed errors \ No newline at end of file diff --git a/wiki/quickstart.md b/wiki/quickstart.md index 66160ba..8eeab96 100644 --- a/wiki/quickstart.md +++ b/wiki/quickstart.md @@ -1,57 +1,83 @@ -# Code-Reducer Technical Overview +# Quickstart -## System Architecture +## Prerequisites -Code-Reducer is a command-line orchestration tool that transforms source code into structured documentation using a Map-Reduce synthesis pattern. The system operates within a local boundary, interacting with the local filesystem and local Large Language Models (LLMs) via the `api/chat` protocol. +- **Ollama** — default LLM backend. Default base URL is `http://localhost:11434` and default model is `ornith:9b`. Override via `OLLAMA_BASE_URL` or `CODE_REDUCER_MODEL_ID` environment variables. +- **Git** — the tool verifies the working directory is inside a git repo on first run. +- **Go** toolchain — the project is a Go module. -### Module Interaction & Data Flow +## Installation -The system architecture is composed of three primary internal layers and external configuration files: +```bash +git clone +cd +go build -o code-reducer ./cmd +``` -#### 1. Orchestration Layer (`cmd`) -Acts as the entry point and lifecycle controller. -* **Validation**: Performs environment checks (git repository existence and working directory identification). -* **State Management**: Evaluates `engine.IsInitialized` to determine if the operation is a full `init` or an incremental `update`. -* **Dispatch**: Validates the environment and passes the `config.Config` to the `engine` package. +Place the binary in your project root or a PATH directory. -#### 2. Configuration Layer (`internal/config`) -Manages hierarchical parameter resolution with the following precedence: -`CLI flags` $\rightarrow$ `Environment variables` $\rightarrow$ `.code-reducer.yaml` $\rightarrow$ `System defaults`. -* **Key Parameters**: `ModelID`, `OllamaBaseURL`, `OllamaNumCtx`, and `DocsDir`. -* **Extraction Steps**: Defines `ExtractionStep` objects mapping specific phases of the synthesis pipeline to prompt templates. +## CLI Commands -#### 3. Map-Reduce Engine (`internal/engine`) -The core computational component responsible for hierarchical content reduction. -* **Discovery & Mapping**: Files are mapped into a `DirNode` tree structure. -* **Change Detection**: Uses `MetadataCache` to compare current file SHA256 fingerprints against previous states. Changes propagate "Affected" status up the `DirNode` tree. -* **Hierarchical Synthesis**: - * **Leaf Processing**: Uses `Chunking` for segments exceeding the context window (using a `0.75` `contextWindowAllocRatio`). - * **Recursion**: If a segment is oversized, recursive reduction is applied. Recursion terminates if the output size is $\ge$ 95% of the input. - * **Aggregation**: Summaries are aggregated from file-level $\rightarrow$ module-level $\rightarrow$ directory-level. +| Command | Purpose | +|---------|---------| +| `setup` | Interactive wizard that writes `.code-reducer.yaml` (model ID, base URL, context size, ignore patterns, docs directory). | +| `init` | First-time documentation build — scans the repo and generates the initial wiki pages. | +| `update` | Incremental regeneration — detects files changed since the last documented commit and rewrites affected wiki pages. | -### External Integration +Persistent flags (`--model-id`, `--num-ctx`) are available on the root command and override config-file values. -* **Documentation Rendering**: `mkdocs.yml` facilitates the deployment of the generated wiki, pulling frontend assets from `cdn.jsdelivr.net` and `cdnjs.cloudflare.com`. -* **Static Analysis**: `sonar-project.properties` manages the exclusion of test files and dependencies during analysis and consumes `coverage.out` reports. +## Configuration ---- +Configuration is resolved in priority order: **CLI flag → environment variable → `.code-reducer.yaml` file**. -## Quickstart Reference +### Environment Variables -### Execution Modes +| Variable | Default | Purpose | +|---|---|---| +| `CODE_REDUCER_MODEL_ID` | `ornith:9b` | LLM model identifier | +| `OLLAMA_BASE_URL` | `http://localhost:11434` | Ollama API endpoint | +| `OLLAMA_NUM_CTX` | `8192` | Context window size | -| Mode | Command | Purpose | -| :--- | :--- | :--- | -| **Initialization** | `init` | Performs a full repository scan and generates initial Markdown documentation. | -| **Incremental** | `update` | Analyzes only files where SHA256 fingerprints have changed relative to the `MetadataCache`. | +### CLI Flags -### Configuration Requirements +| Flag | Purpose | +|---|---| +| `--model-id` | Override LLM model ID | +| `--num-ctx` | Override context window size | -To function, the environment must provide: -1. **Ollama Endpoint**: A running local LLM instance accessible via the configured `OllamaBaseURL`. -2. **Valid Model**: A `ModelID` configured in the `Config` struct. -3. **Context Window**: An integer `OllamaNumCtx` greater than zero. +### Config File -### Data Persistence -* **Documentation**: Synthesized Markdown files are written to the path defined in `DocsDir`. -* **State**: `MetadataCache` (JSON) is maintained in the `docsDir` to enable incremental updates. \ No newline at end of file +`.code-reducer.yaml` (mode `0600`) is created on first run. It stores resolved configuration including model ID, base URL, context size, and custom prompts. + +## Package Structure + +``` +. +├── cmd/ # CLI entry point (init, setup, update) +├── internal/ +│ ├── config/ # Configuration loading, resolution, persistence +│ ├── engine/ # Documentation synthesis pipeline +│ ├── security/ # Path traversal prevention, process locking +│ └── tools/ # Safe filesystem and git utilities +└── wiki/ # Generated documentation output (default) +``` + +## Key Files + +| File | Purpose | +|---|---| +| `.code-reducer.yaml` | Persisted configuration | +| `.metadata.json` | Engine cache (file hashes, module mappings) | +| `.code-reducer.lock` | Process lock file (auto-added to `.gitignore`) | +| `AGENTS.md` | Agent guidelines (updated on init) | +| `architecture.md` | Global architecture document | +| `quickstart.md` | Developer quickstart guide | +| `modules/.md` | Per-module summaries | + +## Error Handling + +- **Lock contention**: `ErrLockHeld` returned if another process holds the lock. +- **Path traversal**: `ErrPathTraversal` returned if input path escapes repo root. +- **LLM failures**: Propagated as wrapped errors. +- **File read failures**: Swallowed and logged via status callback. +- **Git operations**: Errors wrapped and returned (no panics). \ No newline at end of file