diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d3b64a..87d4526 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ All notable changes to this project will be documented in this file. ## Unreleased +- **Binary files are reported, not dumped; the header no longer leaks absolute paths (#21 P11).** A file with NUL bytes in its first 8 KiB (UTF-16/32 text with a BOM excepted) is listed under `## Skipped` as `binary` instead of a `` block in the body (a repo without `.git` produced hundreds of those). The header says `from the `my proj` directory` rather than the absolute path passed with `-d`. - **`--stats[=N]` per-file token breakdown; real token count after a run (#21 P7).** `--stats` lists the largest files by tokens (20 by default; `--stats=0` lists all) and, with `--max-tokens`, which files fit, which is cut first and what is omitted; it writes no document. The `Estimated tokens` line after a run is now the tokenizer's count of the written document (it was bytes/4, off by −11% to −75% on real code); output piped with `-o -` keeps the `~NK` estimate. - **Auto-diff honors `--max-tokens` like the standard renderer (#20 B14).** The auto-diff path used a bytes/4 cut over the finished document (mid-file, ignoring `--encoding`, omissions unreported). Files are now budgeted one by one with the real tokenizer: a file that would cross the budget is clamped at a `--truncate` boundary (and says so) or omitted, and the notice counts the omitted files. The budget search is shared (`markdown::fit_chunk_to_budget`) with the standard path. - **`--structure` lists imports and exports (#22 B15).** The help promised them but only counts were printed (and the internal lists held placeholders). Each file now gets `**Imports:**` / `**Exports:**` lists (Rust `use`/`pub use`, Python, JS/TS `import`/`export`, Go, Java, C/C++ `#include`), capped at 20 entries. Top-level declarations only. diff --git a/src/content_filter.rs b/src/content_filter.rs index 4325225..dde52ee 100644 --- a/src/content_filter.rs +++ b/src/content_filter.rs @@ -222,6 +222,10 @@ pub enum SkipReason { TooLarge, /// Filename (or `.npmrc` / `.pypirc` contents) looks like a credential. Secret, + /// Not a known asset type, but the content is binary (NUL bytes in the + /// first 8 KiB, outside UTF-16/32 text). Rendering it would only print a + /// placeholder block. + Binary, } impl SkipReason { @@ -231,6 +235,7 @@ impl SkipReason { SkipReason::Asset => "asset", SkipReason::TooLarge => "too large", SkipReason::Secret => "secret", + SkipReason::Binary => "binary", } } } @@ -395,9 +400,26 @@ pub fn classify( { return Some((SkipReason::TooLarge, "too large")); } + if looks_binary(path) { + return Some((SkipReason::Binary, "binary")); + } None } +/// Sniff the start of the file. Unreadable files are not reported as binary +/// (the renderer deals with them). +fn looks_binary(path: &Path) -> bool { + use std::io::Read; + let Ok(file) = std::fs::File::open(path) else { + return false; + }; + let mut head = Vec::with_capacity(8192); + if file.take(8192).read_to_end(&mut head).is_err() { + return false; + } + crate::markdown::is_binary_content(&head) +} + /// Stderr lines for a skip list: one warning per secret (path and category /// only), then a single summary line. Empty when nothing was skipped. pub fn report_lines(skipped: &[SkippedFile]) -> Vec { @@ -453,11 +475,13 @@ pub fn summary_line(skipped: &[SkippedFile]) -> Option { let mut assets = 0usize; let mut large = 0usize; let mut secrets = 0usize; + let mut binaries = 0usize; for item in skipped { match item.reason { SkipReason::Asset => assets += 1, SkipReason::TooLarge => large += 1, SkipReason::Secret => secrets += 1, + SkipReason::Binary => binaries += 1, } } let mut parts = Vec::new(); @@ -470,6 +494,9 @@ pub fn summary_line(skipped: &[SkippedFile]) -> Option { if secrets > 0 { parts.push(count_phrase(secrets, "secret", "secrets")); } + if binaries > 0 { + parts.push(count_phrase(binaries, "binary", "binaries")); + } let noun = if skipped.len() == 1 { "file" } else { "files" }; Some(format!( "Skipped {} {} ({}).", diff --git a/src/lib.rs b/src/lib.rs index 0d96457..f184766 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -423,7 +423,7 @@ pub fn run_with_args(args: Args, config: Config, prompter: &impl Prompter) -> io encoding, &format!( "This document contains files from the {} directory with extensions: {} \n", - fences::inline_code(&final_args.input), + fences::inline_code(&markdown::directory_display_name(&final_args.input)), final_args.filter.join(", ") ), ); @@ -432,7 +432,7 @@ pub fn run_with_args(args: Args, config: Config, prompter: &impl Prompter) -> io encoding, &format!( "This document contains all files from the {} directory, optimized for LLM consumption.\n", - fences::inline_code(&final_args.input) + fences::inline_code(&markdown::directory_display_name(&final_args.input)) ), ); } diff --git a/src/markdown.rs b/src/markdown.rs index 063ec9f..a235329 100644 --- a/src/markdown.rs +++ b/src/markdown.rs @@ -113,16 +113,7 @@ pub fn generate_markdown( path: (output_path != "-").then(|| std::path::PathBuf::from(output_path)), }; - let input_dir_name = if input_dir == "." { - let current_dir = std::env::current_dir()?; - current_dir - .file_name() - .and_then(|n| n.to_str()) - .unwrap_or_else(|| current_dir.to_str().unwrap_or("project")) - .to_string() - } else { - input_dir.to_string() - }; + let input_dir_name = directory_display_name(input_dir); // --- Header + file tree (buffered) --- // // Build the header and tree into a buffer first so we can (a) write them in @@ -1109,6 +1100,26 @@ fn first_visibility_warning_for(ext: &str) -> bool { guard.insert(ext.to_string()) } +/// The name shown for the input directory in the document header: its last +/// path component, never the absolute path (which leaks the user's home +/// directory and username into a document that is meant to be shared). +pub(crate) fn directory_display_name(input_dir: &str) -> String { + use std::path::Component; + let path = Path::new(input_dir); + // The lexical name the user typed wins (a symlink keeps its own name). + let last = path.components().next_back(); + if let Some(Component::Normal(name)) = last + && let Some(name) = name.to_str() + { + return name.to_string(); + } + // `.`, `..`, `/`, `C:\`: resolve, and never fall back to a path. + fs::canonicalize(path) + .ok() + .and_then(|p| p.file_name().and_then(|n| n.to_str().map(str::to_string))) + .unwrap_or_else(|| "project".to_string()) +} + /// Bytes read from the start of a file when deciding text vs binary. const BINARY_SNIFF_LEN: usize = 8192; diff --git a/tests/test_binary_and_header.rs b/tests/test_binary_and_header.rs new file mode 100644 index 0000000..3f0ef00 --- /dev/null +++ b/tests/test_binary_and_header.rs @@ -0,0 +1,174 @@ +//! Binary files are reported, not dumped as placeholder blocks, and the header +//! names the directory instead of leaking an absolute path (#21 P11). + +use context_builder::cli::Args; +use context_builder::config::Config; +use context_builder::content_filter::{ContentPolicy, SkipReason, classify}; +use context_builder::{Prompter, run_with_args}; +use std::fs; +use std::path::Path; +use std::process::Command; + +struct Yes; + +impl Prompter for Yes { + fn confirm_overwrite(&self, _file_path: &str) -> std::io::Result { + Ok(true) + } +} + +fn args(input: &Path, output: &Path) -> Args { + Args { + input: input.to_string_lossy().to_string(), + output: output.to_string_lossy().to_string(), + filter: vec![], + ignore: vec![], + line_numbers: false, + preview: false, + token_count: false, + yes: true, + diff_only: false, + clear_cache: false, + encoding: "o200k_base".to_string(), + init: false, + max_tokens: None, + signatures: false, + structure: false, + truncate: "smart".to_string(), + visibility: "all".to_string(), + file_metadata: false, + max_file_size: "256K".to_string(), + hidden: false, + include_secrets: false, + include_lockfiles: false, + stats: None, + } +} + +fn project() -> tempfile::TempDir { + let dir = tempfile::Builder::new() + .prefix("my proj ") + .tempdir() + .unwrap(); + fs::write(dir.path().join("a.txt"), "hello\n").unwrap(); + // Not a known asset extension, but clearly binary. + fs::write(dir.path().join("blob.dat"), [0u8, 1, 2, 3, 0, 0, 255, 254]).unwrap(); + // NUL-padded UTF-16 with a BOM is text. + let mut utf16 = vec![0xFF, 0xFE]; + for c in "wide text\n".encode_utf16() { + utf16.extend_from_slice(&c.to_le_bytes()); + } + fs::write(dir.path().join("wide.txt"), utf16).unwrap(); + dir +} + +#[test] +fn classify_marks_nul_content_as_binary_but_not_utf16_or_plain_text() { + let dir = project(); + let policy = ContentPolicy::new("256K", &[], false); + let verdict = |name: &str| { + let path = dir.path().join(name); + classify(&path, fs::metadata(&path).unwrap().len(), &policy).map(|(r, _)| r) + }; + assert_eq!(verdict("blob.dat"), Some(SkipReason::Binary)); + assert_eq!(verdict("a.txt"), None); + assert_eq!(verdict("wide.txt"), None); + // A path that cannot be opened is left to the renderer. + assert_eq!(classify(&dir.path().join("missing.dat"), 10, &policy), None); +} + +#[test] +fn document_skips_binary_in_the_body_and_lists_it() { + let dir = project(); + let out = dir.path().join("out.md"); + run_with_args(args(dir.path(), &out), Config::default(), &Yes).unwrap(); + let doc = fs::read_to_string(&out).unwrap(); + assert!(doc.contains("- `blob.dat` — binary"), "{doc}"); + assert!(!doc.contains("### File: `blob.dat`"), "{doc}"); + assert!(!doc.contains("