From 7070d46341fd8a2014061ce818df26724347cc95 Mon Sep 17 00:00:00 2001 From: fylorn <249551762+fylorn@users.noreply.github.com> Date: Tue, 13 Oct 2026 16:41:17 +0800 Subject: [PATCH 1/2] fix(redact): search only the written text of a request, located by protocol structure Redaction ran its rules over the whole raw JSON body and then exempted some keys by name (custom rules only). A built-in rule could still match inside a protocol value: with the mobile-number rule on, an 11-digit run inside a Codex reasoning item id (`rs_` + hex) became `<>`, the upstream rejected `input[n].id`, and every later turn of the conversation failed. Outbound redaction now only looks at the text people, models and tools wrote, listed per request shape in the new `tw_dialect::written` module: system prompts, every role's message text, reasoning text and summaries, tool-call arguments and inputs, tool results and tool descriptions, for Anthropic, Chat, Responses, Gemini and Converse requests, plus embeddings, legacy completions, Gemini embeddings and Realtime client events. Ids, types, roles, names, model names, signatures, encrypted content, media and configuration are never searched, and neither are blocks or fields the list does not know. Reasoning text stays on the list: answers are restored before they reach the client, and history sent back has to be replaced again. - `flow::Scope { Written(Shape), Whole }`; `find`, `look`, `look_from`, `look_hits` and `replace` take the request's shape. Rules still run over the raw text once; only when something matches is the body parsed and the matches outside the written strings dropped. The key-name list is gone. - The gateway names the shape from the path (`ClientApi::written`); a converted hop uses the format it was converted to. Unknown paths and WebSocket paths other than Responses and Realtime are not searched. - Stored-body masking and security-log excerpts still mask the whole text. Co-Authored-By: Claude Opus 5.5 --- crates/tw-config/tests/manual/schema.rs | 4 +- crates/tw-control/src/replay.rs | 14 +- crates/tw-dialect/src/caller.rs | 26 +- crates/tw-dialect/src/lib.rs | 1 + crates/tw-dialect/src/written.rs | 849 ++++++++++++++++++ crates/tw-dialect/tests/caller.rs | 17 + crates/tw-gateway/src/bodies.rs | 38 +- crates/tw-gateway/src/client_api.rs | 53 ++ crates/tw-gateway/src/guard.rs | 320 ++++++- crates/tw-gateway/src/guard/detail/tests.rs | 26 +- crates/tw-gateway/src/plugin/bridge.rs | 37 +- .../tw-gateway/src/plugin/reply/tests/mod.rs | 2 +- crates/tw-gateway/src/plugin/request.rs | 4 +- crates/tw-gateway/src/plugin/trial.rs | 2 +- .../tw-gateway/src/plugin/view/tests/mod.rs | 2 +- crates/tw-gateway/src/server/pipeline.rs | 7 +- crates/tw-gateway/src/server/pipeline/hop.rs | 7 +- crates/tw-gateway/src/server/pipeline/plug.rs | 4 +- crates/tw-gateway/src/ws.rs | 26 +- crates/tw-guard/src/policy.rs | 6 +- crates/tw-guard/src/redact/flow.rs | 396 ++++---- crates/tw-guard/tests/scale.rs | 9 +- docs/config.md | 2 +- docs/config.zh-CN.md | 2 +- 24 files changed, 1569 insertions(+), 285 deletions(-) create mode 100644 crates/tw-dialect/src/written.rs diff --git a/crates/tw-config/tests/manual/schema.rs b/crates/tw-config/tests/manual/schema.rs index c00fba8b..b67aa589 100644 --- a/crates/tw-config/tests/manual/schema.rs +++ b/crates/tw-config/tests/manual/schema.rs @@ -1099,8 +1099,8 @@ pub fn sections() -> Vec
{ Kind::Obj("security.redact"), Def::Section, t( - "Outbound redaction (Secrets & privacy in the app): credentials and personal information anywhere in a request are replaced with placeholders before it leaves, or only recorded, rule by rule.", - "出站脱敏（应用里叫「密钥与隐私」）：请求发出前，按每条规则把其中任何位置的凭据和个人信息替换为占位符，或只记录。", + "Outbound redaction (Secrets & privacy in the app): credentials and personal information in the text a request carries (messages, system prompts, reasoning, tool arguments and results, tool descriptions) are replaced with placeholders before it leaves, or only recorded, rule by rule. Identifiers, model names and other protocol fields are sent as they are.", + "出站脱敏（应用里叫「密钥与隐私」）：请求发出前，按每条规则把其中正文（消息、系统提示、推理、工具调用的参数和结果、工具说明）里的凭据和个人信息替换为占位符，或只记录；id、模型名等协议字段原样发出。", ), ), row( diff --git a/crates/tw-control/src/replay.rs b/crates/tw-control/src/replay.rs index 5ae26a49..bb477123 100644 --- a/crates/tw-control/src/replay.rs +++ b/crates/tw-control/src/replay.rs @@ -257,7 +257,8 @@ pub async fn quote( will_redact: { let rt = s.gateway.runtime(); let mode = rt.config.security.redact.mode; - mode.acts() && !tw_gateway::guard::find(mode, &rt.redact, &raw).is_empty() + let shape = tw_gateway::client_api::ClientApi::written(&row.path); + mode.acts() && !tw_gateway::guard::find(mode, &rt.redact, &raw, shape).is_empty() }, pricing_date: book.table().date.clone(), })) @@ -312,8 +313,15 @@ pub async fn run( let rt = s.gateway.runtime(); let ledger = tw_gateway::guard::ledger_for(&raw); let (body, path) = renamed(&row, raw, &model); - let (body, ledger) = - tw_gateway::guard::replace(rt.config.security.redact.mode, &rt.redact, body, &ledger); + // 只在写出来的文字里换，按存下来的那条请求的路径认形状（见 `tw_gateway::guard`） + let shape = tw_gateway::client_api::ClientApi::written(&path); + let (body, ledger) = tw_gateway::guard::replace( + rt.config.security.redact.mode, + &rt.redact, + body, + &ledger, + shape, + ); let url = tw_gateway::forward::upstream_url(&provider.base_url, &path, None); let started = Instant::now(); diff --git a/crates/tw-dialect/src/caller.rs b/crates/tw-dialect/src/caller.rs index d8c9dcba..d357c73a 100644 --- a/crates/tw-dialect/src/caller.rs +++ b/crates/tw-dialect/src/caller.rs @@ -115,26 +115,26 @@ pub fn rewrite(dialect: Dialect, body: &mut Value, mut f: impl FnMut(&mut String } #[derive(Default)] -struct Collect { +pub(crate) struct Collect { at: Vec, - out: Vec, + pub(crate) out: Vec, } impl Collect { - fn key(&mut self, k: &str, f: impl FnOnce(&mut Self)) { + pub(crate) fn key(&mut self, k: &str, f: impl FnOnce(&mut Self)) { self.at.push(Step::Key(k.to_string())); f(self); self.at.pop(); } - fn index(&mut self, i: usize, f: impl FnOnce(&mut Self)) { + pub(crate) fn index(&mut self, i: usize, f: impl FnOnce(&mut Self)) { self.at.push(Step::Index(i)); f(self); self.at.pop(); } /// 眼下这个位置上的值是一个非空字符串就记下 - fn here(&mut self, v: &Value, in_tool_result: bool) { + pub(crate) fn here(&mut self, v: &Value, in_tool_result: bool) { if v.as_str().is_some_and(|s| !s.is_empty()) { self.out.push(Spot { path: self.at.clone(), @@ -144,7 +144,7 @@ impl Collect { } /// `v.k` 是一个非空字符串就记下 - fn field(&mut self, v: &Value, k: &str, in_tool_result: bool) { + pub(crate) fn field(&mut self, v: &Value, k: &str, in_tool_result: bool) { if let Some(x) = v.get(k) { self.key(k, |c| c.here(x, in_tool_result)); } @@ -152,7 +152,7 @@ impl Collect { /// `string | [{text}]` 形状的内容：中间表示用 `text_of` 读它，数组里**每一块**的 /// `text` 都算，不看块的类型 - fn text_of(&mut self, v: &Value, in_tool_result: bool) { + pub(crate) fn text_of(&mut self, v: &Value, in_tool_result: bool) { match v { Value::String(_) => self.here(v, in_tool_result), Value::Array(items) => { @@ -167,7 +167,7 @@ impl Collect { /// 一个值里的全部字符串。中间表示把它整个写成 JSON 文字的地方用（Gemini 的函数 /// 结果、Bedrock 的 `json` 结果）：调用方的字在这些字符串里，键名和 JSON 的标点 /// 不是谁打的字 - fn leaves(&mut self, v: &Value, in_tool_result: bool) { + pub(crate) fn leaves(&mut self, v: &Value, in_tool_result: bool) { match v { Value::String(_) => self.here(v, in_tool_result), Value::Array(items) => { @@ -185,11 +185,11 @@ impl Collect { } } -fn str_of<'a>(v: &'a Value, k: &str) -> Option<&'a str> { +pub(crate) fn str_of<'a>(v: &'a Value, k: &str) -> Option<&'a str> { v.get(k).and_then(Value::as_str) } -fn each<'a>(v: &'a Value, k: &str) -> impl Iterator { +pub(crate) fn each<'a>(v: &'a Value, k: &str) -> impl Iterator { v.get(k) .and_then(Value::as_array) .map(Vec::as_slice) @@ -257,7 +257,7 @@ fn anthropic_block(c: &mut Collect, b: &Value) { /// `document` 块：`text` 来源的 `data`，`content` 来源的各块文字。base64 和 URL /// 来源是文件，不是文字 -fn anthropic_document(c: &mut Collect, b: &Value, in_tool_result: bool) { +pub(crate) fn anthropic_document(c: &mut Collect, b: &Value, in_tool_result: bool) { let Some(src) = b.get("source") else { return; }; @@ -273,7 +273,7 @@ fn anthropic_document(c: &mut Collect, b: &Value, in_tool_result: bool) { } /// `search_result` 块：中间表示里是「标题、来源、正文」三段拼成的一段文字 -fn anthropic_search_result(c: &mut Collect, b: &Value, in_tool_result: bool) { +pub(crate) fn anthropic_search_result(c: &mut Collect, b: &Value, in_tool_result: bool) { c.field(b, "title", in_tool_result); c.field(b, "source", in_tool_result); if let Some(content) = b.get("content") { @@ -394,7 +394,7 @@ fn responses_content(c: &mut Collect, content: &Value, in_tool_result: bool) { /// Gemini 的字段名驼峰和下划线写法都认（`gemini::request::field`）。这个值写在哪个 /// 键下：先驼峰，再下划线 -fn gemini_key(v: &Value, camel: &str) -> Option { +pub(crate) fn gemini_key(v: &Value, camel: &str) -> Option { if v.get(camel).is_some() { return Some(camel.to_string()); } diff --git a/crates/tw-dialect/src/lib.rs b/crates/tw-dialect/src/lib.rs index 563ead38..a33126e4 100644 --- a/crates/tw-dialect/src/lib.rs +++ b/crates/tw-dialect/src/lib.rs @@ -28,3 +28,4 @@ pub mod responses; pub mod think; pub mod url; pub mod usage; +pub mod written; diff --git a/crates/tw-dialect/src/written.rs b/crates/tw-dialect/src/written.rs new file mode 100644 index 00000000..6eb5d081 --- /dev/null +++ b/crates/tw-dialect/src/written.rs @@ -0,0 +1,849 @@ +//! 请求里写出来的文字：人、模型、工具写的字。出站脱敏只看这些。 +//! +//! # 只认名单里的位置 +//! +//! 请求体里还有协议自己的东西：条目和调用的 id、类型、角色、工具名、模型名、推理的签名和 +//! 加密内容、图片、各种配置项。它们的值由协议或上游定，改一个字上游就拒收整个请求 —— +//! Codex 的推理条目 id（`rs_` 加一长串十六进制）里碰巧有 11 位连着的数字，手机号那条规则 +//! 认得出，换成占位符之后这段对话每一轮都被拒。 +//! +//! 所以这里**按协议的结构列出正文在哪儿**，脱敏只在这些字符串里找；名单之外的一律不看， +//! 不认识的块、不认识的字段也不看。协议新加了放正文的地方，在这里补上。 +//! +//! 名单是：系统提示、各个角色的消息文字（模型以前的回答也算）、推理的文字和摘要、工具调用 +//! 的参数、工具结果、工具定义的说明。**推理的文字也算**：回答里的占位符送回客户端之前换回了 +//! 原值，客户端下一轮把整段历史连同推理原样发回来，不再换一次的话，真值就这样发了出去 —— +//! 签过名的推理（Anthropic 的 `thinking`）也只有换回原来的占位符，才和签名对得上。 +//! +//! # 给的是原文里的位置 +//! +//! 脱敏跑在请求体的 JSON 原文上（规则认得 JSON 的转义，换下来的值放回去还是合法的 JSON）， +//! 所以 [`ranges`] 给的是每个正文字符串在原文里引号之间的那一段。哪些字符串算正文在解析 +//! 出来的 [`Value`] 上判断（[`spots`]，和 [`crate::caller`] 一样按位置），再在原文上走一遍、 +//! 按位置对上。 + +use std::collections::HashMap; +use std::ops::Range; + +use serde_json::Value; + +use crate::caller::{ + Collect, Spot, Step, anthropic_document, anthropic_search_result, each, gemini_key, str_of, +}; +use crate::ir::Dialect; + +/// 一个请求体是什么形状：决定正文在哪儿。 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Shape { + /// 生成回答的请求，和同一种形状的数 token、压缩上下文 + Generate(Dialect), + /// OpenAI 的嵌入（`/v1/embeddings`）：`input` + Embeddings, + /// 旧版补全（OpenAI 的 `/v1/completions`、Anthropic 的 `/v1/complete`）：`prompt`、`suffix` + Completion, + /// Gemini 的嵌入（`:embedContent`、`:batchEmbedContents`） + GeminiEmbed, + /// Realtime 连接上客户端发的一帧事件 + RealtimeEvent, +} + +/// 一个 `shape` 形状的请求体里写出来的每一个字符串，按在请求里出现的先后。空串不算； +/// 不是 JSON 对象的请求体没有。 +pub fn spots(shape: Shape, body: &Value) -> Vec { + let mut c = Collect::default(); + if body.is_object() { + match shape { + Shape::Generate(Dialect::Anthropic) => anthropic(&mut c, body), + Shape::Generate(Dialect::Chat) => chat(&mut c, body), + Shape::Generate(Dialect::Responses) => responses(&mut c, body), + Shape::Generate(Dialect::Gemini) => gemini(&mut c, body), + Shape::Generate(Dialect::Bedrock) => bedrock(&mut c, body), + Shape::Embeddings => { + if let Some(input) = body.get("input") { + c.key("input", |c| strings(c, input)); + } + } + Shape::Completion => { + for k in ["prompt", "suffix"] { + if let Some(x) = body.get(k) { + c.key(k, |c| strings(c, x)); + } + } + } + Shape::GeminiEmbed => { + gemini_embed(&mut c, body); + for (i, r) in each(body, "requests") { + c.key("requests", |c| c.index(i, |c| gemini_embed(c, r))); + } + } + Shape::RealtimeEvent => realtime(&mut c, body), + } + } + c.out +} + +/// 原文里每个正文字符串引号之间的那一段，按先后。不是 JSON 的是 `None`。 +/// +/// 字符串里原样写着控制字符（换行、制表符）的请求体也认：有的客户端这么发，上游也收。 +pub fn ranges(shape: Shape, text: &str) -> Option>> { + let value = parse(text)?; + let mut tree = Tree::default(); + for s in spots(shape, &value) { + tree.insert(s.path()); + } + let mut out = Vec::new(); + if !tree.is_empty() { + walk(text, &tree, &mut out); + } + Some(out) +} + +fn parse(text: &str) -> Option { + if let Ok(v) = serde_json::from_str(text) { + return Some(v); + } + if !text.bytes().any(|b| b < 0x20) { + return None; + } + // 控制字符换成空格：都是一个字节，位置不变 + let spaced: String = text + .chars() + .map(|c| if (c as u32) < 0x20 { ' ' } else { c }) + .collect(); + serde_json::from_str(&spaced).ok() +} + +/// 正文字符串的位置，按路径排成一棵树：走原文时一层一层往下对 +#[derive(Default)] +struct Tree { + keys: HashMap, + indices: HashMap, + /// 这个位置本身是一个正文字符串 + leaf: bool, +} + +impl Tree { + fn insert(&mut self, path: &[Step]) { + let mut at = self; + for s in path { + at = match s { + Step::Key(k) => at.keys.entry(k.clone()).or_default(), + Step::Index(i) => at.indices.entry(*i).or_default(), + }; + } + at.leaf = true; + } + + fn is_empty(&self) -> bool { + self.keys.is_empty() && self.indices.is_empty() && !self.leaf + } +} + +/// 原文里正在里面的一层 +enum Frame<'t> { + /// 一个对象，和眼下这个值的键（还没读到键时是 `None`） + Object(Option<&'t Tree>, Option), + /// 一个数组，和眼下这个值的下标 + Array(Option<&'t Tree>, usize), +} + +impl<'t> Frame<'t> { + /// 眼下这个值在树上的位置。不在树上的是 `None`：它里面没有正文 + fn child(&self) -> Option<&'t Tree> { + match self { + Frame::Object(t, Some(k)) => t.and_then(|t| t.keys.get(k)), + Frame::Object(_, None) => None, + Frame::Array(t, i) => t.and_then(|t| t.indices.get(i)), + } + } +} + +/// 走一遍 JSON 原文（已经解析过，是合规的），把树上的字符串在原文里的位置记下来。 +/// +/// **只扫一遍、不建树**：位置一边走一边对，一个不在树上的对象或数组里面什么都不记。 +fn walk(text: &str, root: &Tree, out: &mut Vec>) { + let b = text.as_bytes(); + let mut stack: Vec> = Vec::new(); + let mut i = 0; + while i < b.len() { + match b[i] { + c @ (b'{' | b'[') => { + let at = match stack.last() { + Some(f) => f.child(), + None => Some(root), + }; + stack.push(if c == b'{' { + Frame::Object(at, None) + } else { + Frame::Array(at, 0) + }); + i += 1; + } + b'}' | b']' => { + stack.pop(); + i += 1; + } + b',' => { + match stack.last_mut() { + Some(Frame::Array(_, n)) => *n += 1, + Some(Frame::Object(_, k)) => *k = None, + None => {} + } + i += 1; + } + b'"' => { + let start = i + 1; + let mut j = start; + while j < b.len() && b[j] != b'"' { + j += if b[j] == b'\\' { 2 } else { 1 }; + } + let end = j.min(b.len()); + i = end + 1; + match stack.last_mut() { + // 对象里还没读到键：这一个就是键 + Some(Frame::Object(_, k @ None)) => *k = Some(key(&text[start - 1..i])), + Some(f) if f.child().is_some_and(|t| t.leaf) => out.push(start..end), + _ => {} + } + } + _ => i += 1, + } + } +} + +/// 原文里带引号的一个键，解开转义 +fn key(quoted: &str) -> String { + let inner = "ed[1..quoted.len().saturating_sub(1).max(1)]; + if !inner.contains('\\') { + return inner.to_string(); + } + serde_json::from_str(quoted).unwrap_or_else(|_| inner.to_string()) +} + +/// 一个字符串，或者一组字符串里的每一个（嵌入的 `input`、补全的 `prompt`）。写成 token 的 +/// 数组是数，不是文字 +fn strings(c: &mut Collect, v: &Value) { + match v { + Value::String(_) => c.here(v, false), + Value::Array(items) => { + for (i, item) in items.iter().enumerate() { + c.index(i, |c| c.here(item, false)); + } + } + _ => {} + } +} + +/// `v.k` 下的整个值里的全部字符串：工具调用的参数对象（键是工具定的，值是模型写的） +fn leaves_at(c: &mut Collect, v: &Value, k: &str, in_tool_result: bool) { + if let Some(x) = v.get(k) { + c.key(k, |c| c.leaves(x, in_tool_result)); + } +} + +/// `v.k` 下面一个数组里每一项的 `field` +fn each_field(c: &mut Collect, v: &Value, k: &str, field: &str) { + c.key(k, |c| { + for (i, x) in each(v, k) { + c.index(i, |c| c.field(x, field, false)); + } + }); +} + +// ───────────────────────────────────────────────────────── Anthropic + +/// 系统提示、每个角色的消息、工具的说明 +fn anthropic(c: &mut Collect, v: &Value) { + if let Some(sys) = v.get("system") { + c.key("system", |c| c.text_of(sys, false)); + } + for (i, m) in each(v, "messages") { + let Some(content) = m.get("content") else { + continue; + }; + c.key("messages", |c| { + c.index(i, |c| { + c.key("content", |c| match content { + Value::String(_) => c.here(content, false), + Value::Array(blocks) => { + for (j, b) in blocks.iter().enumerate() { + c.index(j, |c| anthropic_block(c, b)); + } + } + _ => {} + }) + }) + }); + } + each_field(c, v, "tools", "description"); +} + +fn anthropic_block(c: &mut Collect, b: &Value) { + match str_of(b, "type").unwrap_or("") { + "text" => c.field(b, "text", false), + "thinking" => c.field(b, "thinking", false), + "document" => anthropic_document(c, b, false), + "search_result" => anthropic_search_result(c, b, false), + "tool_use" | "server_tool_use" | "mcp_tool_use" => leaves_at(c, b, "input", false), + "tool_result" | "mcp_tool_result" => { + let Some(content) = b.get("content") else { + return; + }; + c.key("content", |c| match content { + Value::String(_) => c.here(content, true), + Value::Array(items) => { + for (k, item) in items.iter().enumerate() { + c.index(k, |c| match str_of(item, "type").unwrap_or("") { + "text" => c.field(item, "text", true), + "document" => anthropic_document(c, item, true), + "search_result" => anthropic_search_result(c, item, true), + _ => {} + }); + } + } + _ => {} + }); + } + _ => {} + } +} + +// ───────────────────────────────────────────────────────── Chat + +/// 每个角色的消息：文字、拒答、推理（`reasoning_content` / `reasoning`）、工具调用的参数， +/// 工具消息的内容；预测输出；工具的说明 +fn chat(c: &mut Collect, v: &Value) { + for (i, m) in each(v, "messages") { + let tool = str_of(m, "role") == Some("tool"); + c.key("messages", |c| { + c.index(i, |c| { + if let Some(content) = m.get("content") { + c.key("content", |c| chat_content(c, content, tool)); + } + for k in ["reasoning_content", "reasoning", "refusal"] { + c.field(m, k, false); + } + c.key("tool_calls", |c| { + for (j, call) in each(m, "tool_calls") { + c.index(j, |c| { + if let Some(f) = call.get("function") { + c.key("function", |c| c.field(f, "arguments", false)); + } + if let Some(x) = call.get("custom") { + c.key("custom", |c| c.field(x, "input", false)); + } + }); + } + }); + // 旧的写法，一条消息一个调用 + if let Some(f) = m.get("function_call") { + c.key("function_call", |c| c.field(f, "arguments", false)); + } + }) + }); + } + if let Some(p) = v.get("prediction") + && let Some(content) = p.get("content") + { + c.key("prediction", |c| { + c.key("content", |c| chat_content(c, content, false)) + }); + } + c.key("tools", |c| { + for (i, t) in each(v, "tools") { + c.index(i, |c| { + for k in ["function", "custom"] { + if let Some(f) = t.get(k) { + c.key(k, |c| c.field(f, "description", false)); + } + } + }); + } + }); +} + +/// 字符串，或者各块的 `text`、`refusal` +fn chat_content(c: &mut Collect, content: &Value, in_tool_result: bool) { + match content { + Value::String(_) => c.here(content, in_tool_result), + Value::Array(parts) => { + for (i, p) in parts.iter().enumerate() { + c.index(i, |c| match str_of(p, "type").unwrap_or("") { + "text" => c.field(p, "text", in_tool_result), + "refusal" => c.field(p, "refusal", in_tool_result), + _ => {} + }); + } + } + _ => {} + } +} + +// ───────────────────────────────────────────────────────── Responses + +/// `instructions`、`input`（一个字符串，或者一条条的条目）、工具的说明 +fn responses(c: &mut Collect, v: &Value) { + c.field(v, "instructions", false); + if let Some(input) = v.get("input") { + c.key("input", |c| match input { + Value::String(_) => c.here(input, false), + Value::Array(items) => { + for (i, item) in items.iter().enumerate() { + c.index(i, |c| responses_item(c, item)); + } + } + _ => {} + }); + } + responses_tools(c, v); +} + +fn responses_item(c: &mut Collect, item: &Value) { + match str_of(item, "type").unwrap_or("message") { + "message" | "agent_message" => { + if let Some(content) = item.get("content") { + c.key("content", |c| responses_content(c, content, false)); + } + } + "reasoning" => { + for k in ["summary", "content"] { + each_field(c, item, k, "text"); + } + } + "function_call" | "mcp_call" => { + c.field(item, "arguments", false); + c.field(item, "output", true); + } + "custom_tool_call" => c.field(item, "input", false), + "tool_search_call" => match item.get("arguments") { + Some(a @ Value::Object(_)) => c.key("arguments", |c| c.leaves(a, false)), + _ => c.field(item, "arguments", false), + }, + "local_shell_call" => { + let Some(action) = item.get("action") else { + return; + }; + c.key("action", |c| { + c.field(action, "working_directory", false); + for k in ["command", "env"] { + leaves_at(c, action, k, false); + } + }); + } + "function_call_output" | "custom_tool_call_output" | "local_shell_call_output" => { + if let Some(output) = item.get("output") { + c.key("output", |c| responses_content(c, output, true)); + } + } + "additional_tools" | "tool_search_output" => responses_tools(c, item), + _ => {} + } +} + +/// 字符串，或者各块里的 `input_text` / `output_text` 的 `text`、`refusal` 的 `refusal` +fn responses_content(c: &mut Collect, content: &Value, in_tool_result: bool) { + match content { + Value::String(_) => c.here(content, in_tool_result), + Value::Array(parts) => { + for (i, p) in parts.iter().enumerate() { + c.index(i, |c| match str_of(p, "type").unwrap_or("") { + "input_text" | "output_text" => c.field(p, "text", in_tool_result), + "refusal" => c.field(p, "refusal", in_tool_result), + _ => {} + }); + } + } + _ => {} + } +} + +/// `tools` 里每个工具的说明，namespace 的说明和它里面的每个工具 +fn responses_tools(c: &mut Collect, v: &Value) { + c.key("tools", |c| { + for (i, t) in each(v, "tools") { + c.index(i, |c| { + c.field(t, "description", false); + if str_of(t, "type") == Some("namespace") { + each_field(c, t, "tools", "description"); + } + }); + } + }); +} + +// ───────────────────────────────────────────────────────── Gemini + +/// 系统提示、每个角色的每一块（文字和推理、函数调用的参数、函数结果、代码执行的代码和输出）、 +/// 函数的说明。数 token 的请求可能把整个生成请求包在 `generateContentRequest` 里 +fn gemini(c: &mut Collect, v: &Value) { + if let Some(k) = gemini_key(v, "systemInstruction") { + let sys = &v[k.as_str()]; + c.key(&k, |c| match sys { + Value::String(_) => c.here(sys, false), + _ => gemini_parts(c, sys), + }); + } + for (i, content) in each(v, "contents") { + c.key("contents", |c| c.index(i, |c| gemini_parts(c, content))); + } + c.key("tools", |c| { + for (i, t) in each(v, "tools") { + let Some(k) = gemini_key(t, "functionDeclarations") else { + continue; + }; + c.index(i, |c| each_field(c, t, &k, "description")); + } + }); + if let Some(k) = gemini_key(v, "generateContentRequest") { + let inner = &v[k.as_str()]; + c.key(&k, |c| gemini(c, inner)); + } +} + +/// 一个 `Content` 的 `parts` +fn gemini_parts(c: &mut Collect, content: &Value) { + c.key("parts", |c| { + for (j, p) in each(content, "parts") { + c.index(j, |c| gemini_part(c, p)); + } + }); +} + +fn gemini_part(c: &mut Collect, p: &Value) { + c.field(p, "text", false); + for (camel, inner, tool) in [ + ("functionCall", "args", false), + ("functionResponse", "response", true), + ] { + if let Some(k) = gemini_key(p, camel) { + let x = &p[k.as_str()]; + c.key(&k, |c| leaves_at(c, x, inner, tool)); + } + } + for (camel, inner, tool) in [ + ("executableCode", "code", false), + ("codeExecutionResult", "output", true), + ] { + if let Some(k) = gemini_key(p, camel) { + let x = &p[k.as_str()]; + c.key(&k, |c| c.field(x, inner, tool)); + } + } +} + +/// 嵌入请求里的 `content` +fn gemini_embed(c: &mut Collect, v: &Value) { + if let Some(content) = v.get("content") { + c.key("content", |c| gemini_parts(c, content)); + } +} + +// ───────────────────────────────────────────────────────── Bedrock + +/// 系统提示、每个角色的每一块（文字、文本文档、推理的文字、工具调用的参数、工具结果）、 +/// 工具的说明 +fn bedrock(c: &mut Collect, v: &Value) { + each_field(c, v, "system", "text"); + for (i, m) in each(v, "messages") { + c.key("messages", |c| { + c.index(i, |c| { + c.key("content", |c| { + for (j, b) in each(m, "content") { + c.index(j, |c| bedrock_block(c, b)); + } + }) + }) + }); + } + if let Some(cfg) = v.get("toolConfig") { + c.key("toolConfig", |c| { + c.key("tools", |c| { + for (i, t) in each(cfg, "tools") { + if let Some(spec) = t.get("toolSpec") { + c.index(i, |c| { + c.key("toolSpec", |c| c.field(spec, "description", false)) + }); + } + } + }) + }); + } +} + +fn bedrock_block(c: &mut Collect, b: &Value) { + c.field(b, "text", false); + if let Some(src) = b.get("document").and_then(|d| d.get("source")) { + c.key("document", |c| { + c.key("source", |c| c.field(src, "text", false)) + }); + } + if let Some(rt) = b + .get("reasoningContent") + .and_then(|r| r.get("reasoningText")) + { + c.key("reasoningContent", |c| { + c.key("reasoningText", |c| c.field(rt, "text", false)) + }); + } + if let Some(u) = b.get("toolUse") { + c.key("toolUse", |c| leaves_at(c, u, "input", false)); + } + if let Some(res) = b.get("toolResult") { + c.key("toolResult", |c| { + c.key("content", |c| { + for (k, item) in each(res, "content") { + c.index(k, |c| { + c.field(item, "text", true); + leaves_at(c, item, "json", true); + }); + } + }) + }); + } +} + +// ───────────────────────────────────────────────────────── Realtime + +/// Realtime 连接上客户端的一帧：会话的指令和工具说明（`session.update`），加进对话的条目 +/// （`conversation.item.create`），要回答时另给的指令和条目（`response.create`） +fn realtime(c: &mut Collect, v: &Value) { + if let Some(s) = v.get("session") { + c.key("session", |c| { + c.field(s, "instructions", false); + each_field(c, s, "tools", "description"); + }); + } + if let Some(item) = v.get("item") { + c.key("item", |c| realtime_item(c, item)); + } + if let Some(r) = v.get("response") { + c.key("response", |c| { + c.field(r, "instructions", false); + each_field(c, r, "tools", "description"); + c.key("input", |c| { + for (i, item) in each(r, "input") { + c.index(i, |c| realtime_item(c, item)); + } + }); + }); + } +} + +fn realtime_item(c: &mut Collect, item: &Value) { + match str_of(item, "type").unwrap_or("") { + "message" => { + c.key("content", |c| { + for (i, p) in each(item, "content") { + c.index(i, |c| { + c.field(p, "text", false); + c.field(p, "transcript", false); + }); + } + }); + } + "function_call" => c.field(item, "arguments", false), + "function_call_output" => c.field(item, "output", true), + _ => {} + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + /// 找到的那些字符串，排好序（`json!` 写出来的键按字母排，原文里的先后跟着变） + fn found(shape: Shape, body: &Value) -> Vec { + let text = body.to_string(); + let mut got: Vec = ranges(shape, &text) + .unwrap() + .into_iter() + .map(|r| text[r].to_string()) + .collect(); + got.sort(); + got + } + + fn sorted(mut v: [&str; N]) -> [&str; N] { + v.sort(); + v + } + + /// Codex 的请求：条目 id、调用 id、类型、角色、推理的加密内容、工具名都不是正文 + #[test] + fn responses_text_is_found_and_protocol_fields_are_not() { + let body = json!({ + "model": "gpt-5.5", + "instructions": "be brief", + "previous_response_id": "resp_1", + "input": [ + {"type": "message", "id": "msg_1", "role": "user", + "content": [{"type": "input_text", "text": "hello"}]}, + {"type": "reasoning", "id": "rs_0a13800138000c", "encrypted_content": "gAAAA", + "summary": [{"type": "summary_text", "text": "thought"}]}, + {"type": "function_call", "id": "fc_1", "call_id": "call_1", "name": "shell", + "arguments": "{\"cmd\":\"ls\"}"}, + {"type": "function_call_output", "call_id": "call_1", "output": "a.rs"}, + {"type": "custom_tool_call", "call_id": "call_2", "name": "apply_patch", "input": "*** Begin"}, + {"type": "message", "role": "assistant", + "content": [{"type": "output_text", "text": "done"}]}, + {"type": "web_search_call", "id": "ws_1", "action": {"query": "q"}}, + ], + "tools": [ + {"type": "function", "name": "shell", "description": "run a command", + "parameters": {"type": "object", "properties": {"cmd": {"type": "string", "description": "the command"}}}}, + {"type": "namespace", "name": "mcp", "description": "MCP tools", + "tools": [{"type": "function", "name": "x", "description": "inner"}]}, + ], + "reasoning": {"effort": "high"}, + }); + assert_eq!( + found(Shape::Generate(Dialect::Responses), &body), + sorted([ + "be brief", + "hello", + "thought", + r#"{\"cmd\":\"ls\"}"#, + "a.rs", + "*** Begin", + "done", + "run a command", + "MCP tools", + "inner", + ]) + ); + } + + #[test] + fn anthropic_covers_every_role_and_leaves_signatures_and_ids_alone() { + let body = json!({ + "model": "claude", + "system": [{"type": "text", "text": "sys"}], + "messages": [ + {"role": "user", "content": "q"}, + {"role": "assistant", "content": [ + {"type": "thinking", "thinking": "hmm", "signature": "sig"}, + {"type": "redacted_thinking", "data": "xxx"}, + {"type": "text", "text": "a"}, + {"type": "tool_use", "id": "toolu_1", "name": "f", "input": {"id": "13800138000", "path": "/x"}}, + ]}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "toolu_1", "content": [{"type": "text", "text": "r"}]}, + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "AAAA"}}, + ]}, + ], + "tools": [{"name": "f", "description": "does f", "input_schema": {"type": "object"}}], + "metadata": {"user_id": "u"}, + }); + assert_eq!( + found(Shape::Generate(Dialect::Anthropic), &body), + sorted(["sys", "q", "hmm", "a", "13800138000", "/x", "r", "does f"]) + ); + } + + #[test] + fn chat_covers_reasoning_tool_calls_and_tool_messages() { + let body = json!({ + "model": "m", + "messages": [ + {"role": "system", "content": "s"}, + {"role": "user", "content": [{"type": "text", "text": "u"}, {"type": "image_url", "image_url": {"url": "data:image/png;base64,AA"}}]}, + {"role": "assistant", "content": null, "reasoning_content": "think", + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "f", "arguments": "{}"}}]}, + {"role": "tool", "tool_call_id": "call_1", "content": "out"}, + ], + "tools": [{"type": "function", "function": {"name": "f", "description": "d", "parameters": {}}}], + }); + assert_eq!( + found(Shape::Generate(Dialect::Chat), &body), + sorted(["s", "u", "think", "{}", "out", "d"]) + ); + } + + #[test] + fn gemini_reads_both_spellings_and_skips_signatures() { + let body = json!({ + "system_instruction": {"parts": [{"text": "sys"}]}, + "contents": [ + {"role": "user", "parts": [{"text": "q"}, {"inlineData": {"mimeType": "image/png", "data": "AA"}}]}, + {"role": "model", "parts": [ + {"text": "t", "thought": true, "thoughtSignature": "sig"}, + {"functionCall": {"name": "f", "args": {"a": "b"}}, "thoughtSignature": "sig2"}, + ]}, + {"role": "user", "parts": [{"function_response": {"name": "f", "response": {"output": "o"}}}]}, + ], + "tools": [{"functionDeclarations": [{"name": "f", "description": "d"}]}], + }); + assert_eq!( + found(Shape::Generate(Dialect::Gemini), &body), + sorted(["sys", "q", "t", "b", "o", "d"]) + ); + } + + #[test] + fn bedrock_covers_reasoning_tools_and_documents() { + let body = json!({ + "system": [{"text": "sys"}], + "messages": [ + {"role": "user", "content": [{"text": "q"}, {"document": {"name": "n", "format": "txt", "source": {"text": "doc"}}}]}, + {"role": "assistant", "content": [ + {"reasoningContent": {"reasoningText": {"text": "hmm", "signature": "sig"}}}, + {"toolUse": {"toolUseId": "tu_1", "name": "f", "input": {"k": "v"}}}, + ]}, + {"role": "user", "content": [{"toolResult": {"toolUseId": "tu_1", "content": [{"text": "r"}, {"json": {"x": "y"}}]}}]}, + ], + "toolConfig": {"tools": [{"toolSpec": {"name": "f", "description": "d", "inputSchema": {"json": {}}}}]}, + }); + assert_eq!( + found(Shape::Generate(Dialect::Bedrock), &body), + sorted(["sys", "q", "doc", "hmm", "v", "r", "y", "d"]) + ); + } + + #[test] + fn the_other_shapes() { + assert_eq!( + found( + Shape::Embeddings, + &json!({"model": "e", "input": ["a", "b"]}) + ), + sorted(["a", "b"]) + ); + assert_eq!( + found( + Shape::Completion, + &json!({"model": "c", "prompt": "p", "suffix": "s"}) + ), + sorted(["p", "s"]) + ); + assert_eq!( + found( + Shape::GeminiEmbed, + &json!({"requests": [{"model": "models/e", "content": {"parts": [{"text": "x"}]}}]}) + ), + sorted(["x"]) + ); + assert_eq!( + found( + Shape::RealtimeEvent, + &json!({"type": "conversation.item.create", "event_id": "evt_1", + "item": {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "hi"}]}}) + ), + sorted(["hi"]) + ); + } + + /// 原文里的位置：转义、重复的键、写成转义的键、字符串里原样的换行 + #[test] + fn ranges_line_up_with_the_raw_text() { + let text = "{\"input\":[{\"type\":\"message\",\"content\":\"a\\\"b\"}],\"\\u0069nstructions\":\"x\ny\",\"model\":\"m\"}"; + let r = ranges(Shape::Generate(Dialect::Responses), text).unwrap(); + let got: Vec<&str> = r.iter().map(|r| &text[r.clone()]).collect(); + assert_eq!(got, ["a\\\"b", "x\ny"]); + assert!(ranges(Shape::Generate(Dialect::Chat), "not json").is_none()); + assert_eq!( + ranges(Shape::Generate(Dialect::Chat), "{\"model\":\"m\"}"), + Some(Vec::new()) + ); + } +} diff --git a/crates/tw-dialect/tests/caller.rs b/crates/tw-dialect/tests/caller.rs index 3e8fa64c..7a410390 100644 --- a/crates/tw-dialect/tests/caller.rs +++ b/crates/tw-dialect/tests/caller.rs @@ -8,6 +8,9 @@ //! 中间表示里调用方的话一个记号都不剩，系统提示和模型的话原样都在 —— 删也删在同一批 //! 字符串上。 //! +//! 出站脱敏的名单（[`tw_dialect::written`]）比调用方的正文宽：中间表示里的系统提示、模型的 +//! 话和推理，在那份名单里也都要找得到（转换时封起来的压缩摘要除外：原文里它是编码过的）。 +//! //! 一处说明：中间表示把 Gemini 的函数结果、Bedrock 的 `json` 结果整个写成 JSON 文字， //! 键名也在里面；这里认的是其中的字符串。记号不放在键名里。 @@ -17,6 +20,7 @@ use serde_json::{Value, json}; use tw_dialect::caller; use tw_dialect::convert::decode; use tw_dialect::ir::{Dialect, Part, Role}; +use tw_dialect::written::{self, Shape}; type Marks = BTreeSet<(String, bool)>; @@ -102,6 +106,19 @@ fn check(d: Dialect, v: Value, users: &[&str], tools: &[&str]) { let kept = not_callers(d, &v); assert!(!kept.is_empty(), "{d:?}: 例子里该有系统提示或模型的话"); + let mut all = Marks::new(); + for s in written::spots(Shape::Generate(d), &v) { + marks_in(s.get(&v).unwrap_or_default(), false, &mut all); + } + let all: BTreeSet = all.into_iter().map(|(m, _)| m).collect(); + for (m, _) in &want { + assert!(all.contains(m), "{d:?}: 脱敏的名单里没有 {m}"); + } + // 转换时封进 `encrypted_content` 的压缩摘要在原文里是编码过的，不是一段字 + let raw = v.to_string(); + for m in kept.iter().filter(|m| raw.contains(m.as_str())) { + assert!(all.contains(m), "{d:?}: 脱敏的名单里没有 {m}"); + } let mut stripped = v.clone(); caller::rewrite(d, &mut stripped, |s, _| { while let (Some(i), Some(j)) = (s.find('«'), s.find('»')) { diff --git a/crates/tw-gateway/src/bodies.rs b/crates/tw-gateway/src/bodies.rs index 70fe22ca..504ea2d0 100644 --- a/crates/tw-gateway/src/bodies.rs +++ b/crates/tw-gateway/src/bodies.rs @@ -542,10 +542,13 @@ mod tests { .with_custom("库", r"postgres://[^\s\x22]+") .unwrap() .recording(Rule::Custom("库".into())); - let body = - r#"{"content":"use postgres://appuser:hunter2@db-primary.internal:5432/orders now"}"#; - let (_, ledger) = - crate::guard::look(tw_config::SecurityMode::Enforce, &rules, body.as_bytes()); + let body = r#"{"messages":[{"content":"use postgres://appuser:hunter2@db-primary.internal:5432/orders now"}]}"#; + let (_, ledger) = crate::guard::look( + tw_config::SecurityMode::Enforce, + &rules, + body.as_bytes(), + crate::guard::TEST_SHAPE, + ); let red = Redaction { rules: Arc::new(rules), ledger, @@ -566,8 +569,12 @@ mod tests { .with_custom("env", r"OPENAI_API_KEY=\S+") .unwrap(); let request = r#"{"messages":[{"role":"user","content":"postgres://app:hunter2@db.internal:5432/orders and OPENAI_API_KEY=abc123xyz"}]}"#; - let (_, ledger) = - crate::guard::look(tw_config::SecurityMode::Enforce, &rules, request.as_bytes()); + let (_, ledger) = crate::guard::look( + tw_config::SecurityMode::Enforce, + &rules, + request.as_bytes(), + crate::guard::TEST_SHAPE, + ); let red = Redaction { rules: Arc::new(rules), ledger, @@ -639,8 +646,12 @@ mod tests { /// 拦截档下这个请求的账本：和 `start` 里一样，按客户端原文编号 fn enforced(body: &str) -> Redaction { let rules = RuleSet::defaults(); - let (_, ledger) = - crate::guard::look(tw_config::SecurityMode::Enforce, &rules, body.as_bytes()); + let (_, ledger) = crate::guard::look( + tw_config::SecurityMode::Enforce, + &rules, + body.as_bytes(), + crate::guard::TEST_SHAPE, + ); Redaction { rules: Arc::new(rules), ledger, @@ -664,6 +675,7 @@ mod tests { &r.rules, Bytes::from(body.clone()), &r.ledger, + crate::guard::TEST_SHAPE, ); let stored = written(record(BodyKind::Request, &body, r)); assert_eq!(stored.as_bytes(), &sent[..]); @@ -688,6 +700,7 @@ mod tests { &r.rules, Bytes::from(body), &r.ledger, + crate::guard::TEST_SHAPE, ); let stored = written(record(BodyKind::Request, body, r)); assert_eq!(stored.as_bytes(), &sent[..]); @@ -708,6 +721,7 @@ mod tests { &r.rules, Bytes::from(body.clone()), &r.ledger, + crate::guard::TEST_SHAPE, ); let stored = written(record(BodyKind::Request, &body, r)); assert!(!stored.contains(KEY), "{stored}"); @@ -777,7 +791,8 @@ mod tests { tw_config::SecurityMode::Enforce, ] { let rules = RuleSet::defaults(); - let seen = crate::guard::look_hits(mode, &rules, body.as_bytes()); + let seen = + crate::guard::look_hits(mode, &rules, body.as_bytes(), crate::guard::TEST_SHAPE); assert_eq!(seen.hits.as_ref().map(Vec::len), Some(2)); let r = Redaction { rules: Arc::new(rules), @@ -869,7 +884,10 @@ mod tests { tw_config::SecurityMode::Enforce, ] { let rules = RuleSet::defaults(); - let seen = crate::guard::look_hits(mode, &rules, body.as_bytes()); + let responses = Some(crate::guard::Shape::Generate( + tw_dialect::ir::Dialect::Responses, + )); + let seen = crate::guard::look_hits(mode, &rules, body.as_bytes(), responses); let r = Redaction { rules: Arc::new(rules), ledger: seen.ledger, diff --git a/crates/tw-gateway/src/client_api.rs b/crates/tw-gateway/src/client_api.rs index 8b72581a..a5d25ed6 100644 --- a/crates/tw-gateway/src/client_api.rs +++ b/crates/tw-gateway/src/client_api.rs @@ -16,6 +16,7 @@ use tw_config::Protocol; use tw_dialect::convert::Decoded; use tw_dialect::ir::{Dialect, Rejection}; +use tw_dialect::written::Shape; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ClientApi { @@ -163,6 +164,25 @@ impl ClientApi { } } + /// 这个路径的请求体是什么形状：出站脱敏只在它写出来的文字里找（见 + /// [`tw_dialect::written`]）。**认不出的路径是 `None`，不找**：不知道哪些字符串是谁写的 + /// 话，换掉的可能是一个 id、一个配置项，上游拒收整个请求。 + /// + /// 生成回答，和请求体同一种形状的数 token、压缩上下文按格式认；嵌入、旧版补全各有各的 + /// 形状。 + pub fn written(path: &str) -> Option { + let api = Self::of_path(path)?; + let p = path.trim_end_matches('/'); + let tail = p.strip_prefix("/v1").unwrap_or(p); + Some(match api { + ClientApi::AnthropicMessages if tail == "/complete" => Shape::Completion, + ClientApi::OpenaiChat if Self::completes(path) => Shape::Completion, + ClientApi::OpenaiChat if Self::embeds(path) => Shape::Embeddings, + ClientApi::Gemini if Self::embeds(path) => Shape::GeminiEmbed, + api => Shape::Generate(api.dialect()), + }) + } + /// 和它同格式的上游协议。 pub fn protocol(&self) -> Protocol { match self { @@ -277,6 +297,39 @@ fn gemini_model(path: &str) -> Option { mod tests { use super::*; + #[test] + fn each_path_says_where_its_written_text_is() { + use Shape::*; + for (path, want) in [ + ("/v1/messages", Some(Generate(Dialect::Anthropic))), + ( + "/v1/messages/count_tokens", + Some(Generate(Dialect::Anthropic)), + ), + ("/v1/complete", Some(Completion)), + ("/v1/chat/completions", Some(Generate(Dialect::Chat))), + ("/v1/completions", Some(Completion)), + ("/v1/embeddings", Some(Embeddings)), + ("/v1/responses", Some(Generate(Dialect::Responses))), + ( + "/backend-api/codex/responses/compact", + Some(Generate(Dialect::Responses)), + ), + ( + "/v1beta/models/g:streamGenerateContent", + Some(Generate(Dialect::Gemini)), + ), + ( + "/v1beta/models/g:countTokens", + Some(Generate(Dialect::Gemini)), + ), + ("/v1beta/models/g:batchEmbedContents", Some(GeminiEmbed)), + ("/v1/files", None), + ] { + assert_eq!(ClientApi::written(path), want, "{path}"); + } + } + #[test] fn each_api_is_recognised_by_its_path() { for (path, want) in [ diff --git a/crates/tw-gateway/src/guard.rs b/crates/tw-gateway/src/guard.rs index 8ac2e50a..abad35ae 100644 --- a/crates/tw-gateway/src/guard.rs +++ b/crates/tw-gateway/src/guard.rs @@ -17,6 +17,13 @@ //! 只有这一份；[`replace`] 在每一跳发出去之前替换 —— 那一跳的请求体可能是 //! 转换过格式的，要换的是真正发出去的那一份。 //! +//! # 只看写出来的文字 +//! +//! 两步都只在请求体里人、模型、工具写出来的文字里找（[`tw_dialect::written`]），按这一份 +//! 请求体的形状认：客户端的原文按路径（[`crate::client_api::ClientApi::written`]），转换过的 +//! 一跳按它转成的格式。认不出形状的（`None`）不找也不换：不知道哪个字符串是谁写的话，换掉的 +//! 可能是一个 id、一个配置项。 +//! //! # 一个值一个占位符，整个请求里都一样 //! //! 拦截档下 [`look`] 按客户端原文里出现的先后给找到的值编好号，每一跳都接着这本账换 @@ -32,19 +39,28 @@ use std::collections::HashSet; use tw_config::SecurityMode as Mode; use tw_guard::content::Screening; use tw_guard::redact::flow; +pub use tw_guard::redact::flow::Shape; use tw_guard::redact::replace::Ledger; use tw_guard::redact::rules::{Finding, Hit, Rule, RuleSet}; pub mod detail; +/// 测试里的请求体大多写成 Anthropic 的样子 +#[cfg(test)] +pub(crate) const TEST_SHAPE: Option = + Some(Shape::Generate(tw_dialect::ir::Dialect::Anthropic)); + /// 按规则找一遍，**不算我们自己的占位符，也不进 base64 载荷**（见 [`flow::hits`]）。 pub fn hits(text: &str, rules: &RuleSet) -> Vec { flow::hits(text, rules) } -/// 找一遍。**观察档和拦截档都找**，关闭时不找（见 [`flow::find`]）。 -pub fn find(mode: Mode, rules: &RuleSet, body: &[u8]) -> Vec { - flow::find(mode, rules, body) +/// 找一遍。**观察档和拦截档都找**，关闭时不找（见 [`flow::find`]）。认不出形状的不找 +pub fn find(mode: Mode, rules: &RuleSet, body: &[u8], shape: Option) -> Vec { + match shape { + Some(shape) => flow::find(mode, rules, body, shape), + None => Vec::new(), + } } /// 一本新账，让开 `body` 里已经写着的占位符（见 [`flow::ledger_for`]）。 @@ -54,53 +70,87 @@ pub fn ledger_for(body: &[u8]) -> Ledger { /// 看一遍客户端发来的原文：报出去的记录，和这个请求的账本（见 [`flow::look`]）。存下来的 /// 那份请求也照这本账换（[`crate::bodies::Redaction`]）。 -pub fn look(mode: Mode, rules: &RuleSet, body: &[u8]) -> (Vec, Ledger) { - flow::look(mode, rules, body) +pub fn look( + mode: Mode, + rules: &RuleSet, + body: &[u8], + shape: Option, +) -> (Vec, Ledger) { + let l = look_hits(mode, rules, body, shape); + (l.found, l.ledger) } /// [`look`]，连同找到的命中（见 [`flow::Look::hits`]）：同一份正文落盘前打码时不再找一遍 /// （[`crate::bodies::BodyRecord::found`]）。 -pub fn look_hits(mode: Mode, rules: &RuleSet, body: &[u8]) -> flow::Look { - flow::look_hits( +pub fn look_hits(mode: Mode, rules: &RuleSet, body: &[u8], shape: Option) -> flow::Look { + look_hits_from( mode, rules, body, + shape, Ledger::new(tw_guard::redact::replace::Scheme::SECRET), ) } /// [`look`]，接着 `seed` 的账编号（见 [`flow::look_from`]）。 -pub fn look_from(mode: Mode, rules: &RuleSet, body: &[u8], seed: Ledger) -> (Vec, Ledger) { - flow::look_from(mode, rules, body, seed) +pub fn look_from( + mode: Mode, + rules: &RuleSet, + body: &[u8], + shape: Option, + seed: Ledger, +) -> (Vec, Ledger) { + let l = look_hits_from(mode, rules, body, shape, seed); + (l.found, l.ledger) } /// [`look_from`]，连同找到的命中（见 [`flow::Look::hits`]）：报出去的每一项要说得出在哪儿 /// （[`detail::secrets`]）。 -pub fn look_hits_from(mode: Mode, rules: &RuleSet, body: &[u8], seed: Ledger) -> flow::Look { - flow::look_hits(mode, rules, body, seed) +/// 认不出形状的不找：和关着一样，什么都没有 +pub fn look_hits_from( + mode: Mode, + rules: &RuleSet, + body: &[u8], + shape: Option, + seed: Ledger, +) -> flow::Look { + match shape { + Some(shape) => flow::look_hits(mode, rules, body, shape, seed), + None => flow::Look { + found: Vec::new(), + ledger: Ledger::new(tw_guard::redact::replace::Scheme::SECRET), + hits: None, + }, + } } /// 拦截档下换掉要发出去的这一份，**接着 `ledger` 的账**（见 [`flow::replace`]）。 +/// 认不出形状的原样发 pub fn replace( mode: Mode, rules: &RuleSet, body: bytes::Bytes, ledger: &Ledger, + shape: Option, ) -> (bytes::Bytes, Ledger) { - flow::replace(mode, rules, body, ledger) + match shape { + Some(shape) => flow::replace(mode, rules, body, ledger, shape), + None => (body, ledger.clone()), + } } /// [`replace`]，`found` 是按这套规则在 `body` 上已经找过的命中（[`look_hits`] 在同一份字节上 -/// 找的）：照它换，不再找一遍。没找过的是 `None`，和 [`replace`] 一样。 +/// 找的，同一个形状）：照它换，不再找一遍。没找过的是 `None`，和 [`replace`] 一样。 pub fn replace_found( mode: Mode, rules: &RuleSet, body: bytes::Bytes, ledger: &Ledger, + shape: Option, found: Option<&[Hit]>, ) -> (bytes::Bytes, Ledger) { let Some(found) = found.filter(|_| mode.acts() && !rules.is_empty()) else { - return replace(mode, rules, body, ledger); + return replace(mode, rules, body, ledger, shape); }; // 只记录的规则命中的不换 let hits = rules.to_replace(found); @@ -110,7 +160,7 @@ pub fn replace_found( } // 找得到命中的一定是 UTF-8（见 `flow::look_hits`） let Ok(text) = std::str::from_utf8(&body) else { - return replace(mode, rules, body, ledger); + return replace(mode, rules, body, ledger, shape); }; let r = tw_guard::redact::replace::apply(text, &hits, ledger.clone()); (bytes::Bytes::from(r.text), r.ledger) @@ -475,7 +525,13 @@ mod tests { #[test] fn enforce_replaces_with_a_placeholder_and_keeps_the_body_valid_json() { - let (out, ledger) = replace(Mode::Enforce, &RuleSet::defaults(), body(), &fresh()); + let (out, ledger) = replace( + Mode::Enforce, + &RuleSet::defaults(), + body(), + &fresh(), + crate::guard::TEST_SHAPE, + ); let text = String::from_utf8(out.to_vec()).unwrap(); assert!(!text.contains(KEY), "{text}"); assert!(text.contains("<>"), "{text}"); @@ -493,19 +549,52 @@ mod tests { #[test] fn observe_finds_what_enforce_would_replace_and_changes_nothing() { // 观察档**只记录，不改变任何行为**；它报的和拦截档会换的是同一批 - let seen = find(Mode::Observe, &RuleSet::defaults(), &body()); + let seen = find( + Mode::Observe, + &RuleSet::defaults(), + &body(), + crate::guard::TEST_SHAPE, + ); assert_eq!(seen.len(), 1); assert!(!seen[0].masked.contains("AAAAAAAAAAAA")); - assert_eq!(seen, find(Mode::Enforce, &RuleSet::defaults(), &body())); - let (out, ledger) = replace(Mode::Observe, &RuleSet::defaults(), body(), &fresh()); + assert_eq!( + seen, + find( + Mode::Enforce, + &RuleSet::defaults(), + &body(), + crate::guard::TEST_SHAPE + ) + ); + let (out, ledger) = replace( + Mode::Observe, + &RuleSet::defaults(), + body(), + &fresh(), + crate::guard::TEST_SHAPE, + ); assert_eq!(out, body()); assert!(ledger.is_empty()); } #[test] fn off_does_not_even_look() { - assert!(find(Mode::Off, &RuleSet::defaults(), &body()).is_empty()); - let (out, _) = replace(Mode::Off, &RuleSet::defaults(), body(), &fresh()); + assert!( + find( + Mode::Off, + &RuleSet::defaults(), + &body(), + crate::guard::TEST_SHAPE + ) + .is_empty() + ); + let (out, _) = replace( + Mode::Off, + &RuleSet::defaults(), + body(), + &fresh(), + crate::guard::TEST_SHAPE, + ); assert_eq!(out, body()); } @@ -513,16 +602,36 @@ mod tests { fn a_binary_body_is_left_alone_instead_of_being_mangled() { // 按字节乱切一个非 UTF-8 的体，得到的是一份坏掉的请求。 let raw = bytes::Bytes::from(vec![0xff, 0xfe, 0x00, 0x01]); - let (out, l) = replace(Mode::Enforce, &RuleSet::defaults(), raw.clone(), &fresh()); + let (out, l) = replace( + Mode::Enforce, + &RuleSet::defaults(), + raw.clone(), + &fresh(), + crate::guard::TEST_SHAPE, + ); assert_eq!(out, raw); assert!(l.is_empty()); - assert!(find(Mode::Enforce, &RuleSet::defaults(), &raw).is_empty()); + assert!( + find( + Mode::Enforce, + &RuleSet::defaults(), + &raw, + crate::guard::TEST_SHAPE + ) + .is_empty() + ); } #[test] fn a_body_with_nothing_to_redact_is_returned_untouched() { let plain = bytes::Bytes::from_static(b"{\"messages\":[]}"); - let (out, l) = replace(Mode::Enforce, &RuleSet::defaults(), plain.clone(), &fresh()); + let (out, l) = replace( + Mode::Enforce, + &RuleSet::defaults(), + plain.clone(), + &fresh(), + crate::guard::TEST_SHAPE, + ); assert_eq!(out, plain); assert!(l.is_empty()); } @@ -545,7 +654,12 @@ mod tests { #[test] fn the_event_items_name_the_rule_and_never_carry_the_value() { let it = items_of( - &find(Mode::Observe, &RuleSet::defaults(), &body()), + &find( + Mode::Observe, + &RuleSet::defaults(), + &body(), + crate::guard::TEST_SHAPE, + ), 0, &body(), ); @@ -572,7 +686,12 @@ mod tests { fn a_request_reports_its_first_hundred_values_and_still_replaces_every_one() { let keys: Vec = (0..150).map(aws_key).collect(); let body = serde_json::json!({"messages": [{"content": keys.join(" ")}]}).to_string(); - let (found, ledger) = look(Mode::Enforce, &RuleSet::defaults(), body.as_bytes()); + let (found, ledger) = look( + Mode::Enforce, + &RuleSet::defaults(), + body.as_bytes(), + crate::guard::TEST_SHAPE, + ); // 找到的一个不少：总共几个不同的值，`found.len()` 说得出来 assert_eq!(found.len(), 150); let it = items_of(&found, 0, body.as_bytes()); @@ -593,7 +712,13 @@ mod tests { ); assert!(items_of(&found, 150, body.as_bytes()).is_empty()); // 没报的照样换掉 - let (out, ledger) = replace(Mode::Enforce, &RuleSet::defaults(), body.into(), &ledger); + let (out, ledger) = replace( + Mode::Enforce, + &RuleSet::defaults(), + body.into(), + &ledger, + crate::guard::TEST_SHAPE, + ); let out = String::from_utf8(out.to_vec()).unwrap(); assert!(!out.contains("AKIA"), "{out}"); assert!(out.contains("<>"), "{out}"); @@ -608,25 +733,46 @@ mod tests { r#"{{"system":"<> {KEY}","messages":[{{"content":"{other} {KEY}"}}]}}"# )); let rules = RuleSet::defaults(); - let seen = look_hits(Mode::Enforce, &rules, &body); + let seen = look_hits(Mode::Enforce, &rules, &body, crate::guard::TEST_SHAPE); let hits = seen.hits.unwrap(); - let again = replace(Mode::Enforce, &rules, body.clone(), &seen.ledger); + let again = replace( + Mode::Enforce, + &rules, + body.clone(), + &seen.ledger, + crate::guard::TEST_SHAPE, + ); let known = replace_found( Mode::Enforce, &rules, body.clone(), &seen.ledger, + crate::guard::TEST_SHAPE, Some(&hits), ); assert_eq!(known.0, again.0); assert_eq!(known.1.table(), again.1.table()); assert!(!String::from_utf8_lossy(&known.0).contains(KEY)); // 观察档不换，找过也不换 - let (out, _) = replace_found(Mode::Observe, &rules, body.clone(), &fresh(), Some(&hits)); + let (out, _) = replace_found( + Mode::Observe, + &rules, + body.clone(), + &fresh(), + crate::guard::TEST_SHAPE, + Some(&hits), + ); assert_eq!(out, body); // 什么都没找到：原样，不拷贝 let plain = bytes::Bytes::from_static(b"{\"messages\":[]}"); - let (out, l) = replace_found(Mode::Enforce, &rules, plain.clone(), &fresh(), Some(&[])); + let (out, l) = replace_found( + Mode::Enforce, + &rules, + plain.clone(), + &fresh(), + crate::guard::TEST_SHAPE, + Some(&[]), + ); assert_eq!(out.as_ptr(), plain.as_ptr()); assert!(l.is_empty()); } @@ -634,17 +780,31 @@ mod tests { /// 查表比和两两比，留下的一模一样：同一条规则下打码一样的算见过，换一条规则就不算 #[test] fn more_found_keeps_exactly_what_comparing_every_pair_kept() { - let text = |range: std::ops::Range| { + let content = |range: std::ops::Range| { let keys: Vec = range.map(aws_key).collect(); - serde_json::json!({ "content": keys.join(" ") }).to_string() + serde_json::json!({ "content": keys.join(" ") }) + }; + let text = |range: std::ops::Range| { + serde_json::json!({ "messages": [content(range)] }).to_string() }; - let mut before = find(Mode::Observe, &RuleSet::defaults(), text(0..60).as_bytes()); + let mut before = find( + Mode::Observe, + &RuleSet::defaults(), + text(0..60).as_bytes(), + crate::guard::TEST_SHAPE, + ); // 前 10 个当成是另一条规则认出来的：打码一样、规则不同，不算见过 for f in &mut before[..10] { f.rule = Rule::Custom(std::sync::Arc::from("aws-again")); } - let after_text = format!("{} {}", text(30..90), text(0..5)); - let after = find(Mode::Observe, &RuleSet::defaults(), after_text.as_bytes()); + let after_text = + serde_json::json!({ "messages": [content(30..90), content(0..5)] }).to_string(); + let after = find( + Mode::Observe, + &RuleSet::defaults(), + after_text.as_bytes(), + crate::guard::TEST_SHAPE, + ); let pairwise: Vec = after .iter() .filter(|f| { @@ -667,18 +827,35 @@ mod tests { let other = "ghp_BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"; let client = format!(r#"{{"system":"{KEY}","messages":[{{"role":"user","content":"{other}"}}]}}"#); - let (found, l0) = look(Mode::Enforce, &RuleSet::defaults(), client.as_bytes()); + let (found, l0) = look( + Mode::Enforce, + &RuleSet::defaults(), + client.as_bytes(), + crate::guard::TEST_SHAPE, + ); assert_eq!((found.len(), l0.len()), (2, 2)); let hop = format!(r#"{{"messages":[{{"role":"user","content":"{other}"}}],"system":"{KEY}"}}"#); - let (out, ledger) = replace(Mode::Enforce, &RuleSet::defaults(), hop.clone().into(), &l0); + let (out, ledger) = replace( + Mode::Enforce, + &RuleSet::defaults(), + hop.clone().into(), + &l0, + crate::guard::TEST_SHAPE, + ); assert_eq!( String::from_utf8(out.to_vec()).unwrap(), r#"{"messages":[{"role":"user","content":"<>"}],"system":"<>"}"# ); assert_eq!(ledger.len(), 2); // 各起一本账的话号就对调了 —— 这条测试防的就是它 - let (alone, _) = replace(Mode::Enforce, &RuleSet::defaults(), hop.into(), &fresh()); + let (alone, _) = replace( + Mode::Enforce, + &RuleSet::defaults(), + hop.into(), + &fresh(), + crate::guard::TEST_SHAPE, + ); assert!( String::from_utf8(alone.to_vec()) .unwrap() @@ -688,13 +865,36 @@ mod tests { #[test] fn look_numbers_only_under_enforce_and_reports_the_same_either_way() { - let (seen, l) = look(Mode::Observe, &RuleSet::defaults(), &body()); - assert_eq!(seen, find(Mode::Observe, &RuleSet::defaults(), &body())); + let (seen, l) = look( + Mode::Observe, + &RuleSet::defaults(), + &body(), + crate::guard::TEST_SHAPE, + ); + assert_eq!( + seen, + find( + Mode::Observe, + &RuleSet::defaults(), + &body(), + crate::guard::TEST_SHAPE + ) + ); assert!(l.is_empty(), "观察档不该编号"); - let (acted, l) = look(Mode::Enforce, &RuleSet::defaults(), &body()); + let (acted, l) = look( + Mode::Enforce, + &RuleSet::defaults(), + &body(), + crate::guard::TEST_SHAPE, + ); assert_eq!(acted, seen); assert_eq!(l.len(), 1); - let (none, l) = look(Mode::Off, &RuleSet::defaults(), &body()); + let (none, l) = look( + Mode::Off, + &RuleSet::defaults(), + &body(), + crate::guard::TEST_SHAPE, + ); assert!(none.is_empty() && l.is_empty()); } @@ -708,9 +908,15 @@ mod tests { .map(|h| t[h.bytes.clone()].to_string()) .collect(); assert_eq!(found, vec!["hunter2".to_string(), KEY.to_string()]); - let body = format!(r#"{{"content":"{t}"}}"#); + let body = format!(r#"{{"messages":[{{"content":"{t}"}}]}}"#); assert_eq!( - find(Mode::Observe, &RuleSet::defaults(), body.as_bytes()).len(), + find( + Mode::Observe, + &RuleSet::defaults(), + body.as_bytes(), + crate::guard::TEST_SHAPE + ) + .len(), 2 ); let (out, _) = replace( @@ -718,6 +924,7 @@ mod tests { &RuleSet::defaults(), body.clone().into(), &ledger_for(body.as_bytes()), + crate::guard::TEST_SHAPE, ); let out = String::from_utf8(out.to_vec()).unwrap(); assert!(out.contains("postgres://app:<>@db/x"), "{out}"); @@ -730,8 +937,19 @@ mod tests { let pasted = format!( r#"{{"messages":[{{"role":"user","content":"上次发的是 <>，这次是 {KEY}"}}]}}"# ); - let (_, l0) = look(Mode::Enforce, &RuleSet::defaults(), pasted.as_bytes()); - let (out, ledger) = replace(Mode::Enforce, &RuleSet::defaults(), pasted.into(), &l0); + let (_, l0) = look( + Mode::Enforce, + &RuleSet::defaults(), + pasted.as_bytes(), + crate::guard::TEST_SHAPE, + ); + let (out, ledger) = replace( + Mode::Enforce, + &RuleSet::defaults(), + pasted.into(), + &l0, + crate::guard::TEST_SHAPE, + ); let out = String::from_utf8(out.to_vec()).unwrap(); assert!(out.contains("这次是 <>"), "{out}"); assert_eq!( @@ -739,14 +957,18 @@ mod tests { format!("<> / {KEY}") ); // 重放用的那本新账也让开它 - let stored = format!("<> {KEY}"); + let stored = format!(r#"{{"messages":[{{"content":"<> {KEY}"}}]}}"#); let (out, _) = replace( Mode::Enforce, &RuleSet::defaults(), bytes::Bytes::from(stored.clone()), &ledger_for(stored.as_bytes()), + crate::guard::TEST_SHAPE, + ); + assert_eq!( + &out[..], + br#"{"messages":[{"content":"<> <>"}]}"# ); - assert_eq!(&out[..], b"<> <>"); } /// 内容过滤的几条规则：拒绝一个词、只记录一个词、删掉零宽字符 diff --git a/crates/tw-gateway/src/guard/detail/tests.rs b/crates/tw-gateway/src/guard/detail/tests.rs index feaa3100..d7341dd7 100644 --- a/crates/tw-gateway/src/guard/detail/tests.rs +++ b/crates/tw-gateway/src/guard/detail/tests.rs @@ -12,6 +12,7 @@ use tw_guard::redact::rules::RuleSet; use super::*; use crate::bodies::Redaction; +use crate::guard::Shape; const KEY: &str = "sk-ant-api03-AAAAAAAAAAAAAAAAAAAAAAAAAA"; const GHP: &str = "ghp_BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"; @@ -39,7 +40,9 @@ fn redacted( rules: RuleSet, ) -> Vec { let text = body.to_string(); - let look = crate::guard::look_hits(mode, &rules, text.as_bytes()); + // 认不出格式的例子写成嵌入请求的样子 + let shape = dialect.map_or(Shape::Embeddings, Shape::Generate); + let look = crate::guard::look_hits(mode, &rules, text.as_bytes(), Some(shape)); let hits = look.hits.unwrap_or_default(); let redaction = Redaction { rules: Arc::new(rules), @@ -270,10 +273,11 @@ fn an_unknown_format_still_has_paths_and_plain_text_has_none() { assert_eq!(place_of(l), (HitPart::Message, None, None, None)); assert_eq!(l.before, "a "); + // 不是 JSON 的一段文字（WebSocket 上别的路径的一帧在日志里的样子）：整段找 let text = format!("plain {KEY} text"); let rules = RuleSet::defaults(); - let look = crate::guard::look_hits(Mode::Observe, &rules, text.as_bytes()); - let hits = look.hits.unwrap(); + let hits = crate::guard::hits(&text, &rules); + let found = tw_guard::redact::rules::findings(&text, &hits); let redaction = Redaction::default(); let seen = Seen { body: text.as_bytes(), @@ -282,7 +286,7 @@ fn an_unknown_format_still_has_paths_and_plain_text_has_none() { redaction: &redaction, replaced: false, }; - let d = secrets(&seen, &look.found); + let d = secrets(&seen, &found); let l = &d[0].locations[0]; assert_eq!((l.path.as_str(), l.before.as_str()), ("", "plain ")); assert_eq!(l.after, " text"); @@ -359,7 +363,12 @@ fn a_value_written_with_other_escapes_shows_its_placeholder() { r#"{{"messages":[{{"role":"user","content":"first {KEY} and postgres://app:hunter\u00e9hunter@db/x end"}}]}}"# ); let rules = RuleSet::defaults(); - let look = crate::guard::look_hits(Mode::Enforce, &rules, text.as_bytes()); + let look = crate::guard::look_hits( + Mode::Enforce, + &rules, + text.as_bytes(), + crate::guard::TEST_SHAPE, + ); let hits = look.hits.unwrap_or_default(); let redaction = Redaction { rules: Arc::new(rules), @@ -1072,7 +1081,12 @@ fn reported_items_on_a_large_body_cost_linear_time() { } let text = json!({"model": "m", "messages": msgs}).to_string(); let rules = RuleSet::defaults(); - let look = crate::guard::look_hits(Mode::Enforce, &rules, text.as_bytes()); + let look = crate::guard::look_hits( + Mode::Enforce, + &rules, + text.as_bytes(), + crate::guard::TEST_SHAPE, + ); let hits = look.hits.unwrap_or_default(); let redaction = Redaction { rules: Arc::new(rules), diff --git a/crates/tw-gateway/src/plugin/bridge.rs b/crates/tw-gateway/src/plugin/bridge.rs index b0a424fa..40f0823d 100644 --- a/crates/tw-gateway/src/plugin/bridge.rs +++ b/crates/tw-gateway/src/plugin/bridge.rs @@ -61,15 +61,20 @@ impl Bridge { /// /// 账里记的是值解开转义之后的样子（见 [`Ledger`]）：插件看到的是解码过的正文，请求体 /// 里写着 `\n` 的私钥块在那儿是真的换行，照样认得出、换成同一个号。 - pub fn learn(&mut self, body: &[u8]) { + /// + /// 和出站脱敏一样只在写出来的文字里找（`shape`，见 [`crate::guard`]）：认不出形状的不找。 + pub fn learn(&mut self, body: &[u8], shape: Option) { let Ok(text) = std::str::from_utf8(body) else { return; }; let seed = std::mem::replace(&mut self.ledger, Ledger::new(Scheme::SECRET)).avoiding(text); - let hits = if self.rules.is_empty() { - Vec::new() - } else { - crate::guard::hits(text, &self.rules) + let hits = match shape.filter(|_| !self.rules.is_empty()) { + Some(shape) => tw_guard::redact::flow::hits_in( + text, + &self.rules, + tw_guard::redact::flow::Scope::Written(shape), + ), + None => Vec::new(), }; self.ledger = if hits.is_empty() { seed @@ -381,6 +386,7 @@ mod tests { serde_json::json!({"messages": [{"content": format!("k={KEY}")}]}) .to_string() .as_bytes(), + crate::guard::TEST_SHAPE, ); let shown = b.hide(&format!("用这把 {KEY} 试试")); assert!(!shown.contains(KEY), "{shown}"); @@ -391,7 +397,10 @@ mod tests { #[test] fn a_key_the_body_did_not_have_is_found_and_numbered_after_the_known_ones() { let mut b = Bridge::new(rules()); - b.learn(format!("{{\"a\":\"{KEY}\"}}").as_bytes()); + b.learn( + format!("{{\"messages\":[{{\"content\":\"{KEY}\"}}]}}").as_bytes(), + crate::guard::TEST_SHAPE, + ); let shown = b.hide(&format!("新的 {OTHER}")); assert_eq!(shown, "新的 <>"); assert_eq!(b.reveal(&shown), format!("新的 {OTHER}")); @@ -404,7 +413,12 @@ mod tests { // 请求体里私钥块的换行是 `\n` 两个字符，插件看到的正文里是真的换行：还是 1 号， // 换回来的是真的换行，不是一个反斜杠和一个 n let mut b = Bridge::new(rules()); - b.learn(serde_json::json!({ "content": PEM }).to_string().as_bytes()); + b.learn( + serde_json::json!({"messages": [{ "content": PEM }]}) + .to_string() + .as_bytes(), + crate::guard::TEST_SHAPE, + ); assert_eq!(b.ledger().len(), 1); let shown = b.hide(&format!("钥匙：{PEM}")); assert_eq!(shown, "钥匙：<>"); @@ -449,7 +463,7 @@ mod tests { #[test] fn nothing_recognised_means_nothing_changes() { let mut b = Bridge::new(rules()); - b.learn(b"{\"x\":\"hello\"}"); + b.learn(b"{\"x\":\"hello\"}", crate::guard::TEST_SHAPE); assert!(b.is_empty()); assert_eq!(b.hide("普通的一段话"), "普通的一段话"); } @@ -457,7 +471,10 @@ mod tests { #[test] fn the_tail_that_could_still_become_a_known_value_is_held() { let mut b = Bridge::new(rules()); - b.learn(format!("{{\"a\":\"{KEY}\"}}").as_bytes()); + b.learn( + format!("{{\"messages\":[{{\"content\":\"{KEY}\"}}]}}").as_bytes(), + crate::guard::TEST_SHAPE, + ); let buf = format!("前面的话 {}", &KEY[..10]); assert_eq!(b.hold_from(&buf), buf.len() - 10); assert_eq!(b.hold_from("别的 sk"), "别的 sk".len() - 2); @@ -579,7 +596,7 @@ mod tests { let body = serde_json::json!({ "messages": messages }); let started = std::time::Instant::now(); let mut b = Bridge::new(rules()); - b.learn(body.to_string().as_bytes()); + b.learn(body.to_string().as_bytes(), crate::guard::TEST_SHAPE); let mut shown = body.clone(); b.hide_value(&mut shown); let text = shown.to_string(); diff --git a/crates/tw-gateway/src/plugin/reply/tests/mod.rs b/crates/tw-gateway/src/plugin/reply/tests/mod.rs index 2552e44c..746d2127 100644 --- a/crates/tw-gateway/src/plugin/reply/tests/mod.rs +++ b/crates/tw-gateway/src/plugin/reply/tests/mod.rs @@ -42,7 +42,7 @@ async fn chain_with( request: &Value, ) -> Chain { let mut bridge = Bridge::new(Arc::new(tw_guard::redact::rules::RuleSet::defaults())); - bridge.learn(request.to_string().as_bytes()); + bridge.learn(request.to_string().as_bytes(), crate::guard::TEST_SHAPE); Chain::start( state, set, diff --git a/crates/tw-gateway/src/plugin/request.rs b/crates/tw-gateway/src/plugin/request.rs index 24211dde..28ffb4fa 100644 --- a/crates/tw-gateway/src/plugin/request.rs +++ b/crates/tw-gateway/src/plugin/request.rs @@ -222,11 +222,11 @@ impl<'a> Hook<'a> { /// 按客户端原文编好号的那本账（第一次调时才编） fn base(&mut self) -> Bridge { - let (rules, body) = (&self.rules, self.body); + let (rules, body, path) = (&self.rules, self.body, self.path); self.base .get_or_insert_with(|| { let mut b = Bridge::new(rules.clone()); - b.learn(body); + b.learn(body, crate::client_api::ClientApi::written(path)); b }) .clone() diff --git a/crates/tw-gateway/src/plugin/trial.rs b/crates/tw-gateway/src/plugin/trial.rs index 2b35252c..5adb6039 100644 --- a/crates/tw-gateway/src/plugin/trial.rs +++ b/crates/tw-gateway/src/plugin/trial.rs @@ -137,7 +137,7 @@ async fn tried( .as_ref() .and_then(|r| serde_json::from_slice::(r.body).ok()); if let Some(r) = &request { - bridge.learn(r.body); + bridge.learn(r.body, crate::client_api::ClientApi::written(r.path)); } let requested = match (&request, dialect) { (Some(r), Some(d)) => super::request::asked_model(d, r.path, parsed.as_ref()), diff --git a/crates/tw-gateway/src/plugin/view/tests/mod.rs b/crates/tw-gateway/src/plugin/view/tests/mod.rs index 4d5dc209..828420d9 100644 --- a/crates/tw-gateway/src/plugin/view/tests/mod.rs +++ b/crates/tw-gateway/src/plugin/view/tests/mod.rs @@ -1200,7 +1200,7 @@ fn placeholders_in_edits_are_revealed_before_write_back() { let mut bridge = Bridge::new(std::sync::Arc::new( tw_guard::redact::rules::RuleSet::defaults(), )); - bridge.learn(raw.to_string().as_bytes()); + bridge.learn(raw.to_string().as_bytes(), crate::guard::TEST_SHAPE); let built = build(Dialect::Anthropic, &raw, MESSAGES).unwrap(); let mut input = trim(&built.view, &all()); bridge.hide_value(&mut input); diff --git a/crates/tw-gateway/src/server/pipeline.rs b/crates/tw-gateway/src/server/pipeline.rs index 95965d43..61f20f39 100644 --- a/crates/tw-gateway/src/server/pipeline.rs +++ b/crates/tw-gateway/src/server/pipeline.rs @@ -1054,7 +1054,12 @@ fn start( /// 找到的命中跟着请求体交去留档：落盘前打码不再把同一份正文找一遍（见 /// [`crate::bodies::BodyRecord::found`]）。 fn look(rt: &Runtime, req: &Inbound) -> tw_guard::redact::flow::Look { - crate::guard::look_hits(rt.config.security.redact.mode, &rt.redact, &req.body) + crate::guard::look_hits( + rt.config.security.redact.mode, + &rt.redact, + &req.body, + crate::client_api::ClientApi::written(req.uri.path()), + ) } /// 这个请求的正文落盘之前怎么换、怎么打码：此刻生效的规则，和这个请求的账本。 diff --git a/crates/tw-gateway/src/server/pipeline/hop.rs b/crates/tw-gateway/src/server/pipeline/hop.rs index ec673082..62ac0e0a 100644 --- a/crates/tw-gateway/src/server/pipeline/hop.rs +++ b/crates/tw-gateway/src/server/pipeline/hop.rs @@ -473,8 +473,13 @@ pub(super) async fn try_upstreams<'a>( // 换成占位符不算：落盘时两边换的是同一个 let differs = out.reshaped || unsealed.as_ptr() != out.body.as_ptr(); let mode = rt.config.security.redact.mode; + // 这一跳发出去的是什么形状：转换过的按转成的格式，直通的按路径 + let shape = match out.target { + Some(d) => Some(crate::guard::Shape::Generate(d)), + None => crate::client_api::ClientApi::written(&out.path), + }; let (body, ledger) = - crate::guard::replace_found(mode, &rt.redact, unsealed, seed, known); + crate::guard::replace_found(mode, &rt.redact, unsealed, seed, shape, known); (body, ledger, differs) }); diff --git a/crates/tw-gateway/src/server/pipeline/plug.rs b/crates/tw-gateway/src/server/pipeline/plug.rs index 06adcf2a..eba78218 100644 --- a/crates/tw-gateway/src/server/pipeline/plug.rs +++ b/crates/tw-gateway/src/server/pipeline/plug.rs @@ -128,7 +128,9 @@ pub(super) async fn attempt( .bridge .as_ref() .map_or_else(|| started.ledger.clone(), |b| b.ledger().clone()); - let look = crate::guard::look_hits_from(mode, &rt.redact, &body, seed); + // 插件改过的这一份按它要发往的路径认形状 + let shape = crate::client_api::ClientApi::written(&c.path); + let look = crate::guard::look_hits_from(mode, &rt.redact, &body, shape, seed); let (found, ledger) = (look.found, look.ledger); let more = crate::guard::more_found(&started.found, found); // 和开头那一条加起来，一个请求报的有上限（见 `crate::guard::REPORTED_MAX`）。位置是插件 diff --git a/crates/tw-gateway/src/ws.rs b/crates/tw-gateway/src/ws.rs index ceb5959b..c1a99d4b 100644 --- a/crates/tw-gateway/src/ws.rs +++ b/crates/tw-gateway/src/ws.rs @@ -643,6 +643,9 @@ struct Pipes { turns: Option, /// 整条连接一行的 Realtime 连接：每一次回答的用量（`response.done`）加到这一行上 realtime: bool, + /// 客户端的帧是什么形状：出站脱敏只在写出来的文字里找（见 [`outbound`]）。Responses 的连接 + /// 每一帧是一个 Responses 请求的样子，Realtime 的是它的事件；别的路径认不出，不找 + written: Option, /// 范围里可能有插件时才有 plugins: Option, /// 每个 `response.create` 发出去的模型名怎么定。Responses 的连接才有 @@ -804,6 +807,13 @@ pub(crate) async fn proxy( let name = &upstream.provider.name; // 每一轮一行的连接连上了：连接本身不留行，每一轮各有各的号（见 `turn`）。连不上的补上 // 这一行，和整条连接一行的一样报 + let written = match &rows { + Rows::Turns { .. } => Some(crate::guard::Shape::Generate( + tw_dialect::ir::Dialect::Responses, + )), + Rows::Connection { realtime: true, .. } => Some(crate::guard::Shape::RealtimeEvent), + Rows::Connection { .. } => None, + }; let (id, mut ending, turns, realtime) = match (rows, &connected) { ( Rows::Connection { @@ -933,6 +943,7 @@ pub(crate) async fn proxy( id, turns, realtime, + written, plugins, naming, requested_model: String::new(), @@ -1497,15 +1508,20 @@ async fn reply_failed(c_tx: &mut ClientSink, err: GatewayError) -> Step { /// `crate::guard::REPORTED_MAX`） fn outbound(state: &AppState, p: &mut Pipes, id: u64, text: String) -> UpMsg { let mode = p.rules.redact_mode; - if !mode.detects() || p.rules.redact.is_empty() { + let Some(shape) = p + .written + .filter(|_| mode.detects() && !p.rules.redact.is_empty()) + else { return UpMsg::Text(text.into()); - } - // 换的和报出去的是同一批：我们自己的占位符、base64 载荷不算（见 `crate::guard::find`） - let hits = crate::guard::hits(&text, &p.rules.redact); + }; + // 换的和报出去的是同一批：只在写出来的文字里找，我们自己的占位符、base64 载荷不算（见 + // `crate::guard::find`） + let scope = tw_guard::redact::flow::Scope::Written(shape); + let hits = tw_guard::redact::flow::hits_in(&text, &p.rules.redact, scope); if hits.is_empty() { return UpMsg::Text(text.into()); } - let found = tw_guard::redact::flow::reported(&text, &p.rules.redact, &hits); + let found = tw_guard::redact::flow::reported(&text, &p.rules.redact, &hits, scope); let replaced = mode.acts().then(|| { // 只记录的规则命中的不换 let r = tw_guard::redact::replace::apply( diff --git a/crates/tw-guard/src/policy.rs b/crates/tw-guard/src/policy.rs index ef9b2558..46bdfca5 100644 --- a/crates/tw-guard/src/policy.rs +++ b/crates/tw-guard/src/policy.rs @@ -1308,19 +1308,21 @@ content: let key = "sk-ant-api03-AAAAAAAAAAAAAAAAAAAAAAAAAA"; let body = format!(r#"{{"messages":[{{"content":"{key} PRJ-123456 E-1234"}}]}}"#); // 找到三处、报三处 - let found = crate::redact::flow::find(Mode::Enforce, &rs, body.as_bytes()); + let shape = crate::redact::flow::Shape::Generate(tw_dialect::ir::Dialect::Anthropic); + let found = crate::redact::flow::find(Mode::Enforce, &rs, body.as_bytes(), shape); assert_eq!(found.len(), 3); // 只换了工号：只记录的原样发，也不进账本 let l = crate::redact::flow::look_hits( Mode::Enforce, &rs, body.as_bytes(), + shape, crate::redact::replace::Ledger::new(crate::redact::replace::Scheme::SECRET), ); assert_eq!(l.ledger.len(), 1); assert_eq!(l.hits.as_ref().map(Vec::len), Some(3)); let (out, ledger) = - crate::redact::flow::replace(Mode::Enforce, &rs, body.clone().into(), &l.ledger); + crate::redact::flow::replace(Mode::Enforce, &rs, body.clone().into(), &l.ledger, shape); let out = String::from_utf8(out.to_vec()).unwrap(); assert!(out.contains(key) && out.contains("PRJ-123456"), "{out}"); assert!( diff --git a/crates/tw-guard/src/redact/flow.rs b/crates/tw-guard/src/redact/flow.rs index 7b3c5c3d..24675cf5 100644 --- a/crates/tw-guard/src/redact/flow.rs +++ b/crates/tw-guard/src/redact/flow.rs @@ -1,17 +1,23 @@ //! 一个请求体在出站脱敏里怎么走：先看一遍、编好号（[`look`]），每一跳发出去之前换 //! （[`replace`]）。两个网关都用这一份。 //! -//! # 查的是整个请求体 +//! # 查的是请求里写出来的文字 //! -//! 不只是用户消息：系统提示、模型以前的回答、工具调用的参数里都可能躺着一把密钥。 -//! 规则直接跑在请求体的 JSON 原文上（见 [`crate::redact::rules::scan`]：它认得 JSON -//! 的转义，换下来的值放回去还是合法的 JSON）。三样东西不算： +//! 不只是用户消息：系统提示、模型以前的回答、推理、工具调用的参数、工具结果里都可能躺着 +//! 一把密钥。**哪些字符串是写出来的文字按协议的结构认**（[`tw_dialect::written`] 的名单）： +//! id、类型、角色、工具名、模型名、签名、配置项这些协议自己的值不找、不换 —— 改一个字上游 +//! 就拒收整个请求，而它们也不是谁写的话。名单之外的（不认识的块、不认识的字段）一律不看。 //! -//! - **base64 载荷**（图片、文件、推理签名、`data:` URI）：一段编码过的图片里碰巧 +//! 规则跑在请求体的 JSON 原文上（见 [`crate::redact::rules::scan`]：它认得 JSON 的转义， +//! 换下来的值放回去还是合法的 JSON），找到的命中只留落在正文字符串里的（[`Scope::Written`]）。 +//! 正文里也有两样不算： +//! +//! - **base64 载荷**（工具参数里的文件内容、文字里的 `data:` URI）：一段编码过的图片里碰巧 //! 有一截像密钥的字母，换掉它换坏的是那张图，报出来的是一条没人看得懂的误报； -//! - **我们自己的占位符**（见 [`hits`]）； -//! - **请求体的结构**：键；自定义规则也不找协议里的 id、类型、角色、工具名、模型名 -//! （见 [`untouched`]）。 +//! - **我们自己的占位符**（见 [`hits_in`]）。 +//! +//! 落盘前打码、日志里的前后文不发给谁，在整段文字上找（[`Scope::Whole`]）：只是不让值留在 +//! 盘上和界面上。 //! //! # 仅记录的规则 //! @@ -33,11 +39,28 @@ use std::ops::Range; +pub use tw_dialect::written::Shape; + use crate::policy::Mode; use crate::redact::replace::{Ledger, Scheme}; use crate::redact::rules::{Finding, Hit, RuleSet}; -/// 按规则找一遍，**不算我们自己的占位符，也不进 base64 载荷**。 +/// 在一份文字的哪些地方找。 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Scope { + /// 一个这种形状的请求体里写出来的文字（[`tw_dialect::written`]）。发出去的请求只在这里 + /// 找、只换这里；不是 JSON 的没有正文可言，一处都不算 + Written(Shape), + /// 整段文字：落盘前打码、日志里的前后文、解码过的一段话。键和 base64 载荷照样不算 + Whole, +} + +/// [`hits_in`] 整段文字（[`Scope::Whole`]）。 +pub fn hits(text: &str, rules: &RuleSet) -> Vec { + hits_in(text, rules, Scope::Whole) +} + +/// 按规则找一遍，**只在 `scope` 里找，不算我们自己的占位符，也不进 base64 载荷**。 /// /// 连接串里写着 `postgres://app:<>@db` 的那一段，在口令那条规则看来就是 /// 一个口令 —— 可它是我们换上去的：存下来的请求拿去重放、用户把详情里看到的东西贴回 @@ -46,25 +69,27 @@ use crate::redact::rules::{Finding, Hit, RuleSet}; /// /// **先去掉、再合并**：每条规则各自的命中先去掉不算的那些，剩下的才合并重叠的（见 /// [`crate::redact::rules::resolve`]）。反过来的话，一处要换的值和一处压在占位符上（或者 -/// 落在协议的 id 里）的命中合成一处之后，会跟着它一起被去掉，原样发出去。 -pub fn hits(text: &str, rules: &RuleSet) -> Vec { +/// 落在正文外面）的命中合成一处之后，会跟着它一起被去掉，原样发出去。 +/// +/// **先在整段上找、再看落在哪儿**：大多数请求一处都不命中，正文和载荷在哪儿等有了命中 +/// 再找 —— 认正文要把请求体解析一遍。 +pub fn hits_in(text: &str, rules: &RuleSet, scope: Scope) -> Vec { let mut hits = crate::redact::rules::scan_unresolved(text, rules); if hits.is_empty() { return hits; } off_placeholders(text, &mut hits); - // 大多数请求一处都不命中：载荷在哪儿等有了命中再找 - if !hits.is_empty() { - let (everyone, custom) = untouched(text); - outside(&mut hits, everyone); - if hits.iter().any(|h| h.rule.custom()) && !custom.is_empty() { - let (mut mine, builtin): (Vec, Vec) = - hits.into_iter().partition(|h| h.rule.custom()); - outside(&mut mine, custom); - hits = builtin; - hits.extend(mine); + if !hits.is_empty() + && let Scope::Written(shape) = scope + { + match tw_dialect::written::ranges(shape, text) { + Some(written) => within(&mut hits, &written), + None => hits.clear(), } } + if !hits.is_empty() { + outside(&mut hits, untouched(text)); + } crate::redact::rules::resolve(hits, rules) } @@ -83,6 +108,15 @@ pub fn hits_plain(text: &str, rules: &RuleSet) -> Vec { crate::redact::rules::plain_with(text, |encoded| hits(encoded, rules)) } +/// 只留整个落在 `spans` 里某一段之内的命中。`spans` 按起点排好、互不重叠（原文里一个个 +/// 字符串引号之间的那一段）。 +fn within(hits: &mut Vec, spans: &[Range]) { + hits.retain(|h| { + let after = spans.partition_point(|s| s.start <= h.bytes.start); + after > 0 && spans[after - 1].end >= h.bytes.end + }); +} + /// 只留和 `spans` 里哪一段都不重叠的命中。 /// /// **不拿每个命中去和每一段比**：一个请求里命中几万处、占位符或载荷又有几万段时（重放 @@ -110,24 +144,25 @@ fn outside(hits: &mut Vec, mut spans: Vec>) { /// 找一遍。**观察档和替换档都找**，关闭时不找。 /// /// **不是 UTF-8 就不看。**图片之类的二进制体里不会有粘贴进来的 key。 -pub fn find(mode: Mode, rules: &RuleSet, body: &[u8]) -> Vec { +pub fn find(mode: Mode, rules: &RuleSet, body: &[u8], shape: Shape) -> Vec { if !mode.detects() || rules.is_empty() { return Vec::new(); } let Ok(text) = std::str::from_utf8(body) else { return Vec::new(); }; - reported(text, rules, &hits(text, rules)) + let scope = Scope::Written(shape); + reported(text, rules, &hits_in(text, rules, scope), scope) } -/// 报出去的记录：`found`（[`hits`] 在 `text` 上找到的），**加上被换成占位符的命中盖住的、 -/// 只记录的规则的命中**。 +/// 报出去的记录：`found`（[`hits_in`] 在 `text` 的 `scope` 里找到的），**加上被换成占位符的 +/// 命中盖住的、只记录的规则的命中**（同一个 `scope` 里找）。 /// /// 一处值两条规则都认得、一条换成占位符一条只记录时，换的那条说了算（见 /// [`crate::redact::rules::scan`]）：发出去的是占位符，`found` 里只记录的那条只剩让开之后的 /// 几截（存下来时照样打码）。报的时候它整个算一处、照样报 —— 用户要看得见它在认哪些值。 /// 没有只记录的规则时就是 `found` 的那些 -pub fn reported(text: &str, rules: &RuleSet, found: &[Hit]) -> Vec { +pub fn reported(text: &str, rules: &RuleSet, found: &[Hit], scope: Scope) -> Vec { let Some(rec) = rules.recording_only() else { return crate::redact::rules::findings(text, found); }; @@ -138,7 +173,7 @@ pub fn reported(text: &str, rules: &RuleSet, found: &[Hit]) -> Vec { .filter(|h| !rules.records(&h.rule)) .cloned() .collect(); - all.extend(hits(text, &rec)); + all.extend(hits_in(text, &rec, scope)); all.sort_by_key(|h| h.bytes.start); crate::redact::rules::findings(text, &all) } @@ -163,15 +198,21 @@ pub fn ledger_for(body: &[u8]) -> Ledger { /// **替换档下账本在这里就编好号**：原文里找到的每个值按出现的先后发号，让开原文里本来 /// 就写着的占位符。之后每一跳都接着这本账换（[`replace`]），存下来的那份请求也照它换。 /// 不在替换档时账本是空的。 -pub fn look(mode: Mode, rules: &RuleSet, body: &[u8]) -> (Vec, Ledger) { - look_from(mode, rules, body, Ledger::new(Scheme::SECRET)) +pub fn look(mode: Mode, rules: &RuleSet, body: &[u8], shape: Shape) -> (Vec, Ledger) { + look_from(mode, rules, body, shape, Ledger::new(Scheme::SECRET)) } /// [`look`]，**接着 `seed` 的账编号**：一个请求改过之后再看一遍时（桌面版的插件改写过的 /// 请求），改过的这一份接着原文那本账编，同一个值还是同一个号，新出现的值接着往后编。 /// 不在替换档时账本是空的，`seed` 用不上。 -pub fn look_from(mode: Mode, rules: &RuleSet, body: &[u8], seed: Ledger) -> (Vec, Ledger) { - let l = look_hits(mode, rules, body, seed); +pub fn look_from( + mode: Mode, + rules: &RuleSet, + body: &[u8], + shape: Shape, + seed: Ledger, +) -> (Vec, Ledger) { + let l = look_hits(mode, rules, body, shape, seed); (l.found, l.ledger) } @@ -191,7 +232,7 @@ pub struct Look { } /// [`look_from`]，把找到的命中一起交回去（见 [`Look::hits`]）。 -pub fn look_hits(mode: Mode, rules: &RuleSet, body: &[u8], seed: Ledger) -> Look { +pub fn look_hits(mode: Mode, rules: &RuleSet, body: &[u8], shape: Shape, seed: Ledger) -> Look { let empty = || Look { found: Vec::new(), ledger: Ledger::new(Scheme::SECRET), @@ -203,8 +244,9 @@ pub fn look_hits(mode: Mode, rules: &RuleSet, body: &[u8], seed: Ledger) -> Look let Ok(text) = std::str::from_utf8(body) else { return empty(); }; - let hits = hits(text, rules); - let found = reported(text, rules, &hits); + let scope = Scope::Written(shape); + let hits = hits_in(text, rules, scope); + let found = reported(text, rules, &hits, scope); let ledger = if !mode.acts() { Ledger::new(Scheme::SECRET) } else { @@ -226,6 +268,7 @@ pub fn replace( rules: &RuleSet, body: bytes::Bytes, ledger: &Ledger, + shape: Shape, ) -> (bytes::Bytes, Ledger) { if !mode.acts() || rules.is_empty() { return (body, ledger.clone()); @@ -234,7 +277,7 @@ pub fn replace( let Ok(text) = std::str::from_utf8(&body) else { return (body, ledger.clone()); }; - let found = hits(text, rules); + let found = hits_in(text, rules, Scope::Written(shape)); // 只记录的规则命中的不换 let hits = rules.to_replace(&found); if hits.is_empty() { @@ -262,25 +305,15 @@ const CARRIERS: &[&str] = &[ /// 图片、签名动辄上千 const PAYLOAD_MIN: usize = 256; -/// 不换的地方，两份：**哪条规则都不换的** —— base64 载荷（见 [`payload`]）和每一个键； -/// **自定义规则不换的** —— 协议里标识一样东西的值（[`structural`]）。 -/// -/// 一个自定义规则 `abc123` 按子串找，会找进 Codex 的条目 id（`fco_019f…`，十六进制的 -/// UUID）里去：换成占位符之后上游以「id 里有别的字符」拒绝整个请求，而历史每一轮都整段 -/// 重发，这段对话从此每一轮都被拒。类型、工具名换掉同理。 -/// -/// **内置规则照找这些值**：它们认的是有前缀、有格式的东西（`AKIA…`、号码），不会长在一个 -/// 十六进制的 id 里；而工具参数这种对象里，凭据和号码常常就写在 `aws_access_key_id`、 -/// `user_id` 这样的键下面。 -fn untouched(text: &str) -> (Vec>, Vec>) { - let (mut everyone, mut custom) = (Vec::new(), Vec::new()); - json_strings(text, |s, key, is_key| match key { - _ if is_key => everyone.push(s), - _ if payload(&text[s.clone()], key) => everyone.push(s), - Some(k) if structural(k) => custom.push(s), - _ => {} +/// 哪条规则都不换的地方：每一个键，和 base64 载荷（见 [`payload`]）。 +fn untouched(text: &str) -> Vec> { + let mut out = Vec::new(); + json_strings(text, |s, key, is_key| { + if is_key || payload(&text[s.clone()], key) { + out.push(s); + } }); - (everyone, custom) + out } /// 一个值是不是 base64 载荷：[`CARRIERS`] 下面、长得像 base64 的，和任何地方的 @@ -289,30 +322,6 @@ fn payload(value: &str, key: Option<&str>) -> bool { data_uri(value) || (key.is_some_and(|k| CARRIERS.contains(&k)) && looks_base64(value)) } -/// 各家协议里值是一个标识、不是一段话的那些键：条目和调用的 id、类型、角色、工具名、 -/// 模型名。**只认协议自己的键**，不按 `_id` 结尾去猜：工具参数里的 `user_id`、`order_id` -/// 是用户的数据。 -fn structural(key: &str) -> bool { - matches!( - key, - "id" | "call_id" - | "item_id" - | "tool_use_id" - | "tool_call_id" - | "toolUseId" - | "previous_response_id" - | "response_id" - | "type" - | "role" - | "name" - | "namespace" - | "model" - | "status" - | "object" - | "phase" - ) -} - /// `end` 是一个字符串的结尾（引号之前）：它后面（跳过空白）是冒号的话，这个字符串是一个键 fn is_key(text: &str, end: usize) -> bool { let b = text.as_bytes(); @@ -397,6 +406,11 @@ fn looks_base64(s: &str) -> bool { mod tests { use super::*; + use tw_dialect::ir::Dialect; + + /// 测试里的请求体大多写成 Anthropic 的样子 + const S: Shape = Shape::Generate(Dialect::Anthropic); + const KEY: &str = "sk-ant-api03-AAAAAAAAAAAAAAAAAAAAAAAAAA"; fn body() -> bytes::Bytes { @@ -411,7 +425,7 @@ mod tests { #[test] fn enforce_replaces_with_a_placeholder_and_keeps_the_body_valid_json() { - let (out, ledger) = replace(Mode::Enforce, &RuleSet::defaults(), body(), &fresh()); + let (out, ledger) = replace(Mode::Enforce, &RuleSet::defaults(), body(), &fresh(), S); let text = String::from_utf8(out.to_vec()).unwrap(); assert!(!text.contains(KEY), "{text}"); assert!(text.contains("<>"), "{text}"); @@ -429,19 +443,19 @@ mod tests { #[test] fn observe_finds_what_enforce_would_replace_and_changes_nothing() { // 观察档**只记录，不改变任何行为**；它报的和替换档会换的是同一批 - let seen = find(Mode::Observe, &RuleSet::defaults(), &body()); + let seen = find(Mode::Observe, &RuleSet::defaults(), &body(), S); assert_eq!(seen.len(), 1); assert!(!seen[0].masked.contains("AAAAAAAAAAAA")); - assert_eq!(seen, find(Mode::Enforce, &RuleSet::defaults(), &body())); - let (out, ledger) = replace(Mode::Observe, &RuleSet::defaults(), body(), &fresh()); + assert_eq!(seen, find(Mode::Enforce, &RuleSet::defaults(), &body(), S)); + let (out, ledger) = replace(Mode::Observe, &RuleSet::defaults(), body(), &fresh(), S); assert_eq!(out, body()); assert!(ledger.is_empty()); } #[test] fn off_does_not_even_look() { - assert!(find(Mode::Off, &RuleSet::defaults(), &body()).is_empty()); - let (out, _) = replace(Mode::Off, &RuleSet::defaults(), body(), &fresh()); + assert!(find(Mode::Off, &RuleSet::defaults(), &body(), S).is_empty()); + let (out, _) = replace(Mode::Off, &RuleSet::defaults(), body(), &fresh(), S); assert_eq!(out, body()); } @@ -449,16 +463,28 @@ mod tests { fn a_binary_body_is_left_alone_instead_of_being_mangled() { // 按字节乱切一个非 UTF-8 的体，得到的是一份坏掉的请求。 let raw = bytes::Bytes::from(vec![0xff, 0xfe, 0x00, 0x01]); - let (out, l) = replace(Mode::Enforce, &RuleSet::defaults(), raw.clone(), &fresh()); + let (out, l) = replace( + Mode::Enforce, + &RuleSet::defaults(), + raw.clone(), + &fresh(), + S, + ); assert_eq!(out, raw); assert!(l.is_empty()); - assert!(find(Mode::Enforce, &RuleSet::defaults(), &raw).is_empty()); + assert!(find(Mode::Enforce, &RuleSet::defaults(), &raw, S).is_empty()); } #[test] fn a_body_with_nothing_to_redact_is_returned_untouched() { let plain = bytes::Bytes::from_static(b"{\"messages\":[]}"); - let (out, l) = replace(Mode::Enforce, &RuleSet::defaults(), plain.clone(), &fresh()); + let (out, l) = replace( + Mode::Enforce, + &RuleSet::defaults(), + plain.clone(), + &fresh(), + S, + ); assert_eq!(out, plain); assert!(l.is_empty()); } @@ -470,18 +496,24 @@ mod tests { let other = "ghp_BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"; let client = format!(r#"{{"system":"{KEY}","messages":[{{"role":"user","content":"{other}"}}]}}"#); - let (found, l0) = look(Mode::Enforce, &RuleSet::defaults(), client.as_bytes()); + let (found, l0) = look(Mode::Enforce, &RuleSet::defaults(), client.as_bytes(), S); assert_eq!((found.len(), l0.len()), (2, 2)); let hop = format!(r#"{{"messages":[{{"role":"user","content":"{other}"}}],"system":"{KEY}"}}"#); - let (out, ledger) = replace(Mode::Enforce, &RuleSet::defaults(), hop.clone().into(), &l0); + let (out, ledger) = replace( + Mode::Enforce, + &RuleSet::defaults(), + hop.clone().into(), + &l0, + S, + ); assert_eq!( String::from_utf8(out.to_vec()).unwrap(), r#"{"messages":[{"role":"user","content":"<>"}],"system":"<>"}"# ); assert_eq!(ledger.len(), 2); // 各起一本账的话号就对调了 —— 这条测试防的就是它 - let (alone, _) = replace(Mode::Enforce, &RuleSet::defaults(), hop.into(), &fresh()); + let (alone, _) = replace(Mode::Enforce, &RuleSet::defaults(), hop.into(), &fresh(), S); assert!( String::from_utf8(alone.to_vec()) .unwrap() @@ -491,13 +523,13 @@ mod tests { #[test] fn look_numbers_only_under_enforce_and_reports_the_same_either_way() { - let (seen, l) = look(Mode::Observe, &RuleSet::defaults(), &body()); - assert_eq!(seen, find(Mode::Observe, &RuleSet::defaults(), &body())); + let (seen, l) = look(Mode::Observe, &RuleSet::defaults(), &body(), S); + assert_eq!(seen, find(Mode::Observe, &RuleSet::defaults(), &body(), S)); assert!(l.is_empty(), "观察档不该编号"); - let (acted, l) = look(Mode::Enforce, &RuleSet::defaults(), &body()); + let (acted, l) = look(Mode::Enforce, &RuleSet::defaults(), &body(), S); assert_eq!(acted, seen); assert_eq!(l.len(), 1); - let (none, l) = look(Mode::Off, &RuleSet::defaults(), &body()); + let (none, l) = look(Mode::Off, &RuleSet::defaults(), &body(), S); assert!(none.is_empty() && l.is_empty()); } @@ -505,20 +537,27 @@ mod tests { #[test] fn look_from_goes_on_numbering_from_the_seed() { let other = "ghp_BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB"; - let (_, seed) = look(Mode::Enforce, &RuleSet::defaults(), &body()); + let (_, seed) = look(Mode::Enforce, &RuleSet::defaults(), &body(), S); let changed = format!(r#"{{"messages":[{{"content":"{other} 和 {KEY}"}}]}}"#); let (found, ledger) = look_from( Mode::Enforce, &RuleSet::defaults(), changed.as_bytes(), + S, seed.clone(), ); assert_eq!(found.len(), 2); - let (out, _) = replace(Mode::Enforce, &RuleSet::defaults(), changed.into(), &ledger); + let (out, _) = replace( + Mode::Enforce, + &RuleSet::defaults(), + changed.into(), + &ledger, + S, + ); let out = String::from_utf8(out.to_vec()).unwrap(); assert!(out.contains("<> 和 <>"), "{out}"); // 观察档不编号，种子也用不上 - let (_, l) = look_from(Mode::Observe, &RuleSet::defaults(), &body(), seed); + let (_, l) = look_from(Mode::Observe, &RuleSet::defaults(), &body(), S, seed); assert!(l.is_empty()); } @@ -531,7 +570,7 @@ mod tests { r#"{{"system":"<> {KEY}","messages":[{{"content":"{other} {KEY}"}}]}}"# ); let rules = RuleSet::defaults(); - let l = look_hits(Mode::Enforce, &rules, body.as_bytes(), fresh()); + let l = look_hits(Mode::Enforce, &rules, body.as_bytes(), S, fresh()); let found = hits(&body, &rules); assert_eq!(found.len(), 3); assert_eq!(l.hits.as_deref(), Some(&found[..])); @@ -539,21 +578,21 @@ mod tests { assert_eq!(l.ledger.table(), replaced.ledger.table()); assert_eq!(l.ledger.len(), 2); assert_eq!((l.found.clone(), l.ledger.table().clone()), { - let (f, ledger) = look(Mode::Enforce, &rules, body.as_bytes()); + let (f, ledger) = look(Mode::Enforce, &rules, body.as_bytes(), S); (f, ledger.table().clone()) }); // 观察档照样找、照样交回命中，只是不编号 - let o = look_hits(Mode::Observe, &rules, body.as_bytes(), fresh()); + let o = look_hits(Mode::Observe, &rules, body.as_bytes(), S, fresh()); assert_eq!((o.found, o.hits), (l.found, l.hits)); assert!(o.ledger.is_empty()); // 关着的、不是 UTF-8 的不找 assert!( - look_hits(Mode::Off, &rules, body.as_bytes(), fresh()) + look_hits(Mode::Off, &rules, body.as_bytes(), S, fresh()) .hits .is_none() ); assert!( - look_hits(Mode::Enforce, &rules, &[0xff, 0xfe], fresh()) + look_hits(Mode::Enforce, &rules, &[0xff, 0xfe], S, fresh()) .hits .is_none() ); @@ -569,9 +608,9 @@ mod tests { .map(|h| t[h.bytes.clone()].to_string()) .collect(); assert_eq!(found, vec!["hunter2".to_string(), KEY.to_string()]); - let body = format!(r#"{{"content":"{t}"}}"#); + let body = format!(r#"{{"messages":[{{"content":"{t}"}}]}}"#); assert_eq!( - find(Mode::Observe, &RuleSet::defaults(), body.as_bytes()).len(), + find(Mode::Observe, &RuleSet::defaults(), body.as_bytes(), S).len(), 2 ); let (out, _) = replace( @@ -579,6 +618,7 @@ mod tests { &RuleSet::defaults(), body.clone().into(), &ledger_for(body.as_bytes()), + S, ); let out = String::from_utf8(out.to_vec()).unwrap(); assert!(out.contains("postgres://app:<>@db/x"), "{out}"); @@ -591,8 +631,8 @@ mod tests { let pasted = format!( r#"{{"messages":[{{"role":"user","content":"上次发的是 <>，这次是 {KEY}"}}]}}"# ); - let (_, l0) = look(Mode::Enforce, &RuleSet::defaults(), pasted.as_bytes()); - let (out, ledger) = replace(Mode::Enforce, &RuleSet::defaults(), pasted.into(), &l0); + let (_, l0) = look(Mode::Enforce, &RuleSet::defaults(), pasted.as_bytes(), S); + let (out, ledger) = replace(Mode::Enforce, &RuleSet::defaults(), pasted.into(), &l0, S); let out = String::from_utf8(out.to_vec()).unwrap(); assert!(out.contains("这次是 <>"), "{out}"); assert_eq!( @@ -600,14 +640,18 @@ mod tests { format!("<> / {KEY}") ); // 重放用的那本新账也让开它 - let stored = format!("<> {KEY}"); + let stored = format!(r#"{{"messages":[{{"content":"<> {KEY}"}}]}}"#); let (out, _) = replace( Mode::Enforce, &RuleSet::defaults(), bytes::Bytes::from(stored.clone()), &ledger_for(stored.as_bytes()), + S, + ); + assert_eq!( + &out[..], + br#"{"messages":[{"content":"<> <>"}]}"# ); - assert_eq!(&out[..], b"<> <>"); } /// 一段 base64 图片里碰巧有一截像密钥：报出来是没人看得懂的误报，换掉它换坏的是那张图 @@ -639,6 +683,7 @@ mod tests { &RuleSet::defaults(), body.clone().into(), &fresh(), + S, ); let v: serde_json::Value = serde_json::from_slice(&out).unwrap(); let parts = &v["messages"][0]["content"]; @@ -647,71 +692,88 @@ mod tests { assert_eq!(parts[3]["signature"], blob.as_str()); } - /// 自定义规则按子串找：`abc123` 会找进 Codex 的条目 id（十六进制的 UUID）里去，换成 - /// 占位符之后上游以「id 里有别的字符」拒绝整个请求，这段对话从此每一轮都被拒 + /// 只找写出来的文字。协议自己的值一个字都不动：Codex 推理条目的 id（`rs_` 加十六进制） + /// 里碰巧有 11 位连着的数字，手机号那条认得出；自定义规则 `abc123` 按子串找，会找进条目 + /// id、工具名、模型名里 —— 换掉哪一个，上游都拒收整个请求，这段对话从此每一轮都被拒 #[test] - fn keys_and_identifiers_are_left_alone() { - let rules = RuleSet::defaults().with_custom("mine", "abc123").unwrap(); + fn only_the_written_text_is_looked_at() { + let rules = RuleSet::only(&["cn-mobile-phone"]) + .with_custom("mine", "abc123") + .unwrap(); let body = serde_json::json!({ "model": "abc123-model", "previous_response_id": "resp_abc123", "abc123": "a key that happens to match", + "instructions": "say abc123", "input": [ {"type": "message", "id": "msg_019fabc123de", "role": "user", - "content": [{"type": "input_text", "text": "my code is abc123"}]}, - {"type": "function_call", "id": "fc_abc123", "call_id": "call_abc123", - "name": "abc123", "arguments": "{\"q\":\"abc123\"}"}, - {"type": "function_call_output", "id": "fco_019f93de-abc123", "call_id": "call_abc123", + "content": [{"type": "input_text", "text": "my code is abc123, call 13800138000"}]}, + {"type": "reasoning", "id": "rs_0a1b2c13800138000c9d8e", "encrypted_content": "gAAAAabc123", + "summary": [{"type": "summary_text", "text": "the user said abc123"}]}, + {"type": "function_call", "id": "fc_abc123", "call_id": "call_13800138000", "name": "abc123", + "namespace": "abc123", "arguments": "{\"q\":\"abc123\"}"}, + {"type": "function_call_output", "id": "fco_019f93de-abc123", "call_id": "call_13800138000", "output": "found abc123"}, + {"type": "web_search_call", "id": "ws_abc123", "action": {"query": "abc123"}}, ], - "messages": [{"role": "assistant", "content": [ - {"type": "tool_use", "id": "toolu_abc123", "name": "x", "input": {}}, - {"type": "tool_result", "tool_use_id": "toolu_abc123", "content": "abc123"}, - ]}], - "toolUse": {"toolUseId": "abc123"}, + "tools": [{"type": "function", "name": "abc123", "description": "finds abc123", + "parameters": {"type": "object", "properties": {"abc123": {"type": "string"}}}}], }) .to_string(); - let (out, _) = replace(Mode::Enforce, &rules, body.into(), &fresh()); + let shape = Shape::Generate(Dialect::Responses); + let (out, _) = replace(Mode::Enforce, &rules, body.into(), &fresh(), shape); let v: serde_json::Value = serde_json::from_slice(&out).unwrap(); assert_eq!(v["model"], "abc123-model"); assert_eq!(v["previous_response_id"], "resp_abc123"); assert_eq!(v["abc123"], "a key that happens to match"); + assert_eq!(v["instructions"], "say <>"); let input = &v["input"]; assert_eq!(input[0]["id"], "msg_019fabc123de"); - assert_eq!(input[0]["content"][0]["text"], "my code is <>"); - assert_eq!(input[1]["id"], "fc_abc123"); - assert_eq!(input[1]["call_id"], "call_abc123"); - assert_eq!(input[1]["name"], "abc123"); - assert_eq!(input[1]["arguments"], "{\"q\":\"<>\"}"); - assert_eq!(input[2]["id"], "fco_019f93de-abc123"); - assert_eq!(input[2]["output"], "found <>"); - let parts = &v["messages"][0]["content"]; - assert_eq!(parts[0]["id"], "toolu_abc123"); - assert_eq!(parts[1]["tool_use_id"], "toolu_abc123"); - assert_eq!(parts[1]["content"], "<>"); - assert_eq!(v["toolUse"]["toolUseId"], "abc123"); + assert_eq!( + input[0]["content"][0]["text"], + "my code is <>, call <>" + ); + assert_eq!(input[1]["id"], "rs_0a1b2c13800138000c9d8e"); + assert_eq!(input[1]["encrypted_content"], "gAAAAabc123"); + assert_eq!( + input[1]["summary"][0]["text"], + "the user said <>" + ); + assert_eq!(input[2]["id"], "fc_abc123"); + assert_eq!(input[2]["call_id"], "call_13800138000"); + assert_eq!(input[2]["name"], "abc123"); + assert_eq!(input[2]["namespace"], "abc123"); + assert_eq!(input[2]["arguments"], "{\"q\":\"<>\"}"); + assert_eq!(input[3]["id"], "fco_019f93de-abc123"); + assert_eq!(input[3]["call_id"], "call_13800138000"); + assert_eq!(input[3]["output"], "found <>"); + // 不认识的条目不看 + assert_eq!(input[4]["action"]["query"], "abc123"); + assert_eq!(v["tools"][0]["name"], "abc123"); + assert_eq!(v["tools"][0]["description"], "finds <>"); + assert!(v["tools"][0]["parameters"]["properties"]["abc123"].is_object()); } - /// 工具参数这种对象里，凭据和号码就写在 `aws_access_key_id`、`user_id` 下面：内置规则 - /// 照找；自定义规则也照找不属于协议的 `*_id` + /// 工具调用的参数是模型写的：凭据和号码写在什么键下面都找，叫 `id` 的也一样。请求的 + /// `metadata` 是配置，不是谁写的话，不找 #[test] - fn credentials_under_id_like_keys_of_user_data_are_still_found() { + fn tool_call_arguments_are_looked_at_whatever_their_keys() { let rules = RuleSet::defaults().with_custom("工号", "E-\\d{6}").unwrap(); let key = "AKIAABCDEFGHIJKLMNOP"; let body = serde_json::json!({ "messages": [{"role": "assistant", "content": [ {"type": "tool_use", "id": "toolu_1", "name": "deploy", - "input": {"aws_access_key_id": key, "employee_id": "E-123456"}}, + "input": {"aws_access_key_id": key, "id": "E-123456"}}, ]}], "metadata": {"user_id": key}, }) .to_string(); - let (out, _) = replace(Mode::Enforce, &rules, body.into(), &fresh()); + let (out, _) = replace(Mode::Enforce, &rules, body.into(), &fresh(), S); let v: serde_json::Value = serde_json::from_slice(&out).unwrap(); let input = &v["messages"][0]["content"][0]["input"]; assert_eq!(input["aws_access_key_id"], "<>"); - assert_eq!(input["employee_id"], "<>"); - assert_eq!(v["metadata"]["user_id"], "<>"); + assert_eq!(input["id"], "<>"); + assert_eq!(v["metadata"]["user_id"], key); assert_eq!(v["messages"][0]["content"][0]["id"], "toolu_1"); } @@ -785,8 +847,8 @@ mod tests { }; let send = |p: &RedactPolicy, body: &str| { let rules = p.rules().unwrap(); - let l = look_hits(Mode::Enforce, &rules, body.as_bytes(), fresh()); - let (out, _) = replace(Mode::Enforce, &rules, body.to_string().into(), &l.ledger); + let l = look_hits(Mode::Enforce, &rules, body.as_bytes(), S, fresh()); + let (out, _) = replace(Mode::Enforce, &rules, body.to_string().into(), &l.ledger, S); let reported: Vec<(String, bool)> = l .found .iter() @@ -923,8 +985,8 @@ mod tests { bare.actions.clear(); let out = |p: &RedactPolicy| { let rules = p.rules().unwrap(); - let l = look_hits(Mode::Enforce, &rules, body.as_bytes(), fresh()); - let (out, _) = replace(Mode::Enforce, &rules, body.clone().into(), &l.ledger); + let l = look_hits(Mode::Enforce, &rules, body.as_bytes(), S, fresh()); + let (out, _) = replace(Mode::Enforce, &rules, body.clone().into(), &l.ledger, S); String::from_utf8(out.to_vec()).unwrap() }; assert_eq!(out(&p), out(&bare), "{text:?} {p:?}"); @@ -962,8 +1024,8 @@ mod tests { "命中要排好、互不重叠：{all:?}" ); // 发出去的照样是 JSON，回答里原样复述的话能一字不差地换回来 - let l = look_hits(Mode::Enforce, &rules, body.as_bytes(), fresh()); - let (sent, ledger) = replace(Mode::Enforce, &rules, body.clone().into(), &l.ledger); + let l = look_hits(Mode::Enforce, &rules, body.as_bytes(), S, fresh()); + let (sent, ledger) = replace(Mode::Enforce, &rules, body.clone().into(), &l.ledger, S); serde_json::from_slice::(&sent).expect("still JSON"); assert_eq!( crate::redact::replace::restore(std::str::from_utf8(&sent).unwrap(), &ledger), @@ -980,10 +1042,13 @@ mod tests { .unwrap() .with_custom("右", "bbbccc") .unwrap(); - let body = r#"{"content":"x aaabbbccc y"}"#; - let l = look_hits(Mode::Enforce, &rules, body.as_bytes(), fresh()); - let (out, ledger) = replace(Mode::Enforce, &rules, body.into(), &l.ledger); - assert_eq!(&out[..], br#"{"content":"x <> y"}"#); + let body = r#"{"messages":[{"content":"x aaabbbccc y"}]}"#; + let l = look_hits(Mode::Enforce, &rules, body.as_bytes(), S, fresh()); + let (out, ledger) = replace(Mode::Enforce, &rules, body.into(), &l.ledger, S); + assert_eq!( + &out[..], + br#"{"messages":[{"content":"x <> y"}]}"# + ); assert_eq!( crate::redact::replace::restore(std::str::from_utf8(&out).unwrap(), &ledger), body @@ -997,7 +1062,7 @@ mod tests { .with_custom("库", r"postgres://[^\s\x22]+") .unwrap(); rules = rules.recording(crate::redact::rules::Rule::Custom("库".into())); - let body = r#"{"content":"use postgres://app:hunter2@db.internal:5432/orders now"}"#; + let body = r#"{"messages":[{"content":"use postgres://app:hunter2@db.internal:5432/orders now"}]}"#; let found = hits(body, &rules); // 口令那一截换成占位符，前后两截还是只记录的那条规则的，存下来时打码 let pieces: Vec<(&str, bool)> = found @@ -1012,21 +1077,20 @@ mod tests { ("@db.internal:5432/orders", true) ] ); - let (out, _) = replace(Mode::Enforce, &rules, body.into(), &fresh()); + let (out, _) = replace(Mode::Enforce, &rules, body.into(), &fresh(), S); assert_eq!( &out[..], - br#"{"content":"use postgres://app:<>@db.internal:5432/orders now"}"# + br#"{"messages":[{"content":"use postgres://app:<>@db.internal:5432/orders now"}]}"# ); // 报的时候只记录的那条整个算一处 - let reported: Vec = reported(body, &rules, &found) + let reported: Vec = reported(body, &rules, &found, Scope::Written(S)) .iter() .map(|f| f.rule.id().to_string()) .collect(); assert_eq!(reported, ["库", "conn-string-password"]); } - /// 先去掉压在占位符上的、落在协议 id 里的命中，再合并：一处要换的值不会跟着一处该去掉的 - /// 命中一起被去掉 + /// 先去掉压在占位符上的命中，再合并：一处要换的值不会跟着一处该去掉的命中一起被去掉 #[test] fn hits_that_do_not_count_are_dropped_before_overlaps_merge() { use crate::policy::{ContentMatch, CustomRedactRule, RedactAction, RedactPolicy}; @@ -1041,8 +1105,8 @@ mod tests { }; let send = |p: &RedactPolicy, body: &str| { let rules = p.rules().unwrap(); - let l = look_hits(Mode::Enforce, &rules, body.as_bytes(), fresh()); - let (out, _) = replace(Mode::Enforce, &rules, body.to_string().into(), &l.ledger); + let l = look_hits(Mode::Enforce, &rules, body.as_bytes(), S, fresh()); + let (out, _) = replace(Mode::Enforce, &rules, body.to_string().into(), &l.ledger, S); String::from_utf8(out.to_vec()).unwrap() }; // 邮箱和一处一直连到已经写着的占位符上的自定义命中 @@ -1062,18 +1126,6 @@ mod tests { }; let body = r#"{"messages":[{"role":"user","content":"use postgres://app:hunter2@db.internal/x<> now"}]}"#; assert!(!send(&p, body).contains("hunter2")); - // 工具调用的 id 里的一把内置认得的密钥，和一处更早开始的自定义命中交叠：自定义规则不碰 - // 协议的 id，内置规则照换 - let key = "sk-proj-AbCdEfGhIjKlMnOpQrStUvWxYz0123456789abcdEFGH"; - let p = RedactPolicy { - custom: vec![custom(r"x sk-proj")], - ..Default::default() - }; - let body = serde_json::json!({"messages":[{"role":"assistant","tool_calls":[ - {"id": format!("x {key}"), "type":"function","function":{"name":"f","arguments":"{}"}}]}]}) - .to_string(); - let out = send(&p, &body); - assert!(!out.contains(key), "{out}"); } /// 自定义规则的正则（`\S+`）从结构上的字开始匹配、或者一路吃过引号：只换字符串里面的那一段， @@ -1083,8 +1135,8 @@ mod tests { let rules = RuleSet::none().with_custom("all", r"\S+").unwrap(); let body = r#"{"messages":[{"role":"user","content":"a-b c"},{"role":"user","content":"d"}]}"#; - let l = look_hits(Mode::Enforce, &rules, body.as_bytes(), fresh()); - let (out, ledger) = replace(Mode::Enforce, &rules, body.into(), &l.ledger); + let l = look_hits(Mode::Enforce, &rules, body.as_bytes(), S, fresh()); + let (out, ledger) = replace(Mode::Enforce, &rules, body.into(), &l.ledger, S); let v: serde_json::Value = serde_json::from_slice(&out).expect("still JSON"); assert_eq!( v["messages"][0]["content"], diff --git a/crates/tw-guard/tests/scale.rs b/crates/tw-guard/tests/scale.rs index d22f648a..9f9a504b 100644 --- a/crates/tw-guard/tests/scale.rs +++ b/crates/tw-guard/tests/scale.rs @@ -30,6 +30,9 @@ fn key(i: usize) -> String { format!("AKIA{tail}") } +/// 请求体都写成 Anthropic 的样子 +const SHAPE: flow::Shape = flow::Shape::Generate(Dialect::Anthropic); + /// 一条用户消息，里面是 `n` 个不同的密钥 fn request(n: usize) -> String { let keys: Vec = (0..n).map(key).collect(); @@ -42,10 +45,10 @@ fn round_trip(n: usize) -> Duration { let body = request(n); let started = Instant::now(); - let (found, ledger) = flow::look(Mode::Enforce, &rules, body.as_bytes()); + let (found, ledger) = flow::look(Mode::Enforce, &rules, body.as_bytes(), SHAPE); assert_eq!(found.len(), n, "一个不同的值一条"); assert_eq!(ledger.len(), n); - let (sent, ledger) = flow::replace(Mode::Enforce, &rules, body.clone().into(), &ledger); + let (sent, ledger) = flow::replace(Mode::Enforce, &rules, body.clone().into(), &ledger, SHAPE); let sent = String::from_utf8(sent.to_vec()).unwrap(); assert!(!sent.contains("AKIA"), "每一个都换掉了"); assert!(sent.contains(&format!("<>"))); @@ -172,7 +175,7 @@ fn nested_runs(k: usize) -> Duration { let rules = RuleSet::none().with_custom("ids", r"\d{6,}").unwrap(); let content: String = (6..6 + k).map(|l| format!("{} ", "1".repeat(l))).collect(); let body = json!({"messages": [{"role": "user", "content": content}]}).to_string(); - let (_, ledger) = flow::look(Mode::Enforce, &rules, body.as_bytes()); + let (_, ledger) = flow::look(Mode::Enforce, &rules, body.as_bytes(), SHAPE); assert_eq!(ledger.len(), k); let started = Instant::now(); let out = replace::hide_values(&body, &ledger, replace::Form::Json); diff --git a/docs/config.md b/docs/config.md index 65200a72..df23ec80 100644 --- a/docs/config.md +++ b/docs/config.md @@ -749,7 +749,7 @@ refuses, deletes, replaces or only records. | Field | Type | Default | Description | |---|---|---|---| -| `redact` | object, [`security.redact`](#cfg-security-redact) | — | Outbound redaction (Secrets & privacy in the app): credentials and personal information anywhere in a request are replaced with placeholders before it leaves, or only recorded, rule by rule. | +| `redact` | object, [`security.redact`](#cfg-security-redact) | — | Outbound redaction (Secrets & privacy in the app): credentials and personal information in the text a request carries (messages, system prompts, reasoning, tool arguments and results, tool descriptions) are replaced with placeholders before it leaves, or only recorded, rule by rule. Identifiers, model names and other protocol fields are sent as they are. | | `inspect_tools` | object, [`security.inspect_tools`](#cfg-security-inspect_tools) | — | Tool-call inspection (Dangerous commands in the app): dangerous commands in the tool calls a model returns cut the response off. | | `content` | object, [`security.content`](#cfg-security-content) | — | Content filter: words, patterns or characters (hidden ones among them) in what the caller sends, and words or patterns in the text of the answer; each rule refuses the request, deletes or replaces what it matched, or only records it. | diff --git a/docs/config.zh-CN.md b/docs/config.zh-CN.md index f038a2b1..89df5779 100644 --- a/docs/config.zh-CN.md +++ b/docs/config.zh-CN.md @@ -554,7 +554,7 @@ pricing: | 字段 | 类型 | 默认值 | 说明 | |---|---|---|---| -| `redact` | 对象，见 [`security.redact`](#cfg-security-redact) | — | 出站脱敏（应用里叫「密钥与隐私」）：请求发出前，按每条规则把其中任何位置的凭据和个人信息替换为占位符，或只记录。 | +| `redact` | 对象，见 [`security.redact`](#cfg-security-redact) | — | 出站脱敏（应用里叫「密钥与隐私」）：请求发出前，按每条规则把其中正文（消息、系统提示、推理、工具调用的参数和结果、工具说明）里的凭据和个人信息替换为占位符，或只记录；id、模型名等协议字段原样发出。 | | `inspect_tools` | 对象，见 [`security.inspect_tools`](#cfg-security-inspect_tools) | — | 工具调用审查（应用里叫「危险命令」）：模型返回的工具调用中出现危险命令时切断响应。 | | `content` | 对象，见 [`security.content`](#cfg-security-content) | — | 内容过滤：调用方发送的内容中、或模型回答的正文中出现指定的词、写法或字符（包括隐藏字符）时，按规则拒绝请求、删除或替换命中的内容，或只记录。 | From fe9f52c5d3100266aec78177ba5542d6b89b30e2 Mon Sep 17 00:00:00 2001 From: fylorn <249551762+fylorn@users.noreply.github.com> Date: Tue, 13 Oct 2026 17:06:13 +0800 Subject: [PATCH 2/2] test(ws): frames carry their text in input, as a Responses connection sends it Co-Authored-By: Claude Opus 5.5 --- crates/tw-gateway/tests/ws.rs | 54 ++++++++++++++--------------------- 1 file changed, 22 insertions(+), 32 deletions(-) diff --git a/crates/tw-gateway/tests/ws.rs b/crates/tw-gateway/tests/ws.rs index 840dab9f..d70d174a 100644 --- a/crates/tw-gateway/tests/ws.rs +++ b/crates/tw-gateway/tests/ws.rs @@ -170,6 +170,14 @@ async fn start_gateway(up: SocketAddr, mode: SecurityMode, inspect: SecurityMode addr } +/// 客户端在 Responses 的连接上说的一句话：写在 `input` 里。出站脱敏只看写出来的文字， +/// 一帧光秃秃的字不是 Responses 的写法，不看 +fn said(text: &str) -> tokio_tungstenite::tungstenite::Message { + tokio_tungstenite::tungstenite::Message::Text( + serde_json::json!({ "input": text }).to_string().into(), + ) +} + async fn connect( gw: SocketAddr, ) -> tokio_tungstenite::WebSocketStream> { @@ -192,11 +200,9 @@ async fn a_secret_in_a_frame_is_redacted_before_it_reaches_the_upstream() { let gw = start_gateway(up, SecurityMode::Enforce, SecurityMode::Observe).await; let mut c = connect(gw).await; - c.send(tokio_tungstenite::tungstenite::Message::Text( - format!("我的 key 是 {USER_KEY}，帮我看看").into(), - )) - .await - .unwrap(); + c.send(said(&format!("我的 key 是 {USER_KEY}，帮我看看"))) + .await + .unwrap(); let back = tokio::time::timeout(Duration::from_secs(3), c.next()) .await .expect("等回帧超时") @@ -224,7 +230,7 @@ async fn a_private_key_echoed_in_a_frame_keeps_the_frame_json() { let gw = start_gateway(up, SecurityMode::Enforce, SecurityMode::Observe).await; let mut c = connect(gw).await; - let frame = serde_json::json!({ "text": format!("钥匙：{pem}") }).to_string(); + let frame = serde_json::json!({ "input": format!("钥匙：{pem}") }).to_string(); c.send(tokio_tungstenite::tungstenite::Message::Text( frame.clone().into(), )) @@ -243,7 +249,7 @@ async fn a_private_key_echoed_in_a_frame_keeps_the_frame_json() { let text = back.into_text().unwrap(); assert_eq!(text.as_str(), frame); let v: serde_json::Value = serde_json::from_str(&text).expect("回来的一帧还是 JSON"); - assert_eq!(v["text"], format!("钥匙：{pem}")); + assert_eq!(v["input"], format!("钥匙：{pem}")); } /// 一个占位符切在上游的两帧里，客户端拿到的照样是原值：正文是真的换行，参数解出来也是。 @@ -253,11 +259,7 @@ async fn a_placeholder_split_across_two_frames_comes_back_whole() { let (up, _) = start_upstream("split").await; let gw = start_gateway(up, SecurityMode::Enforce, SecurityMode::Observe).await; let mut c = connect(gw).await; - c.send(tokio_tungstenite::tungstenite::Message::Text( - serde_json::json!({ "text": pem }).to_string().into(), - )) - .await - .unwrap(); + c.send(said(pem)).await.unwrap(); let mut got: Vec = Vec::new(); while !got .iter() @@ -308,19 +310,13 @@ async fn a_held_half_placeholder_survives_the_ledger_growing() { let (up, _) = start_upstream("grow").await; let gw = start_gateway(up, SecurityMode::Enforce, SecurityMode::Observe).await; let mut c = connect(gw).await; - c.send(tokio_tungstenite::tungstenite::Message::Text( - format!("第一把 {USER_KEY}").into(), - )) - .await - .unwrap(); + c.send(said(&format!("第一把 {USER_KEY}"))).await.unwrap(); // 扣住的那一半之前的字先到：这时网关已经扣着 `<