From cd24eea972d993ecb0fd889031ad135cf140eada Mon Sep 17 00:00:00 2001 From: zsd Date: Wed, 14 Oct 2026 17:50:53 +0800 Subject: [PATCH] =?UTF-8?q?feat(jev-sim):=20=E6=96=B0=E5=A2=9E=E4=BC=9A?= =?UTF-8?q?=E8=AF=9D=E5=86=85=E5=86=B3=E7=AD=96=E6=A8=A1=E6=8B=9F=E6=8F=92?= =?UTF-8?q?=E4=BB=B6=EF=BC=88Choice/Noul/Score=EF=BC=8C=E8=BD=AE=E8=BD=AC?= =?UTF-8?q?=E5=8E=BB=E5=81=8F=E5=B7=AE=20+=20NONE=20=E5=87=BA=E5=8F=A3=20+?= =?UTF-8?q?=20=E6=B8=A9=E5=BA=A6=E7=BC=A9=E6=94=BE=E6=A0=A1=E5=87=86?= =?UTF-8?q?=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../jev-sim/.zcode-plugin/plugin.json | 11 + external_plugins/jev-sim/README.md | 103 ++++++++ external_plugins/jev-sim/README_CN.md | 103 ++++++++ .../jev-sim/eval/choice-eval.jsonl | 12 + external_plugins/jev-sim/scripts/aggregate.py | 219 ++++++++++++++++++ external_plugins/jev-sim/scripts/calibrate.py | 131 +++++++++++ .../jev-sim/skills/jev-sim/SKILL.md | 138 +++++++++++ marketplace.json | 86 +++++-- 8 files changed, 789 insertions(+), 14 deletions(-) create mode 100644 external_plugins/jev-sim/.zcode-plugin/plugin.json create mode 100644 external_plugins/jev-sim/README.md create mode 100644 external_plugins/jev-sim/README_CN.md create mode 100644 external_plugins/jev-sim/eval/choice-eval.jsonl create mode 100644 external_plugins/jev-sim/scripts/aggregate.py create mode 100644 external_plugins/jev-sim/scripts/calibrate.py create mode 100644 external_plugins/jev-sim/skills/jev-sim/SKILL.md diff --git a/external_plugins/jev-sim/.zcode-plugin/plugin.json b/external_plugins/jev-sim/.zcode-plugin/plugin.json new file mode 100644 index 00000000..1b94b8d7 --- /dev/null +++ b/external_plugins/jev-sim/.zcode-plugin/plugin.json @@ -0,0 +1,11 @@ +{ + "name": "jev-sim", + "version": "0.2.0", + "description": "Jev-style decision simulation on the session model: Choice/Noul/Score typed decisions with rotation debiasing, an explicit NONE outlet, three-state output, and temperature-scaling calibration", + "description_i18n": { + "en": "Jev-style decision simulation on the session model: Choice/Noul/Score typed decisions with rotation debiasing, an explicit NONE outlet, three-state output, and temperature-scaling calibration", + "zh-CN": "用会话内大模型模拟 Jev 决策模型：Choice/Noul/Score 类型化决策，轮转去偏差、NONE 出口、三态输出、温度缩放校准" + }, + "author": { "name": "forrestneo" }, + "skills": "./skills" +} diff --git a/external_plugins/jev-sim/README.md b/external_plugins/jev-sim/README.md new file mode 100644 index 00000000..0092a0e9 --- /dev/null +++ b/external_plugins/jev-sim/README.md @@ -0,0 +1,103 @@ +# jev-sim: an in-session decision model + +Simulates [Jev](https://typesafe.ai)-style typed decisions with the LLM (GLM) already running in your ZCode session. Turns judgment questions into probabilistic, machine-readable decisions — no external decision-model API, no API key, no data leaves your machine. + +## Three primitives + +| Primitive | Input | Output | Typical use | +|---|---|---|---| +| Choice | state + question + options | probability per option + NONE outlet | classification, routing, branching | +| Noul | state + proposition | P(true) in 0-1 | boolean judgment, risk screening | +| Score | state + rubric | level distribution + expected value | ranking, quality rating | + +## How it works + +Jev scores each candidate via token probabilities from sentence completion. Session-model APIs expose no token probabilities (the GLM API has no logprobs), so the protocol compensates with mechanisms validated in the open decision-model community: + +1. **Letter readout** — each judge subagent outputs exactly one JSON line (options encoded A-Z) +2. **Option rotation** (cyclic shift, AnyJev's L0 layer) — ask the same question K=3 times with rotated option order, removing position bias; vote disagreement is itself signal +3. **Symmetric framings** (Noul) — positive / reversed / evidence-first views, removing framing bias +4. **Frequency as probability** — K votes, smoothed and normalized into a raw distribution +5. **Temperature-scaling calibration** — fit temperature T on labeled items, `p_cal ∝ p_raw^(1/T)`; instruction models are overconfident, and this step translates "vote unanimity" into "how often this much confidence was historically right" + +## Output contract + +```json +{ + "primitive": "choice", + "protocol": "jev-sim/0.2", + "status": "answer", + "question": "Which team should handle this email?", + "decision": { "type": "choice", "choice": "billing-support", "probability": 0.72 }, + "probabilities": [ + { "option": "billing-support", "probability": 0.72 }, + { "option": "legal", "probability": 0.14 }, + { "option": "sales", "probability": 0.14 } + ], + "none_probability": 0.02, + "confidence": 0.72, + "threshold": 0.6, + "abstained": false, + "calibrated": true +} +``` + +`status` is one of: **answer** (normal decision), **none** (NONE votes won — no option holds, refuse to force a pick), **clarify** (top probability below threshold; ask for more context or escalate). Harness-level errors are reported as errors, never disguised as decisions. + +## Usage + +After installing, start a new task, select the jev-sim skill in the composer, and hand it a judgment question: + +> State: customer email — broken chair on arrival, no support response for three days, threatens a consumer-complaint hotline and a lawyer. +> Question: which team should handle this? Options: marketing / billing-support / legal / sales + +**Use for**: classification, routing, boolean judgment, rubric scoring, risk screening. +**Do not use for**: open-ended writing, exact symbol/string lookups, arithmetic, or anything expressible as a deterministic rule — just do those directly. + +Calibration (recommended once before first use): `python scripts/calibrate.py --results eval/your-results.jsonl`. It writes `scripts/calibration.json`, applied automatically afterwards. `eval/choice-eval.jsonl` ships a 12-item seed set; see the "calibration workflow" section in SKILL.md. + +## Side effects and dependencies + +- **Network access: none.** All sampling happens inside the current ZCode session; no external API is called, no data is sent anywhere. +- **Model dependency:** uses the session's built-in LLM (judge subagents via the Agent tool); no extra account or key. +- **Command execution:** runs `python scripts/aggregate.py` and `scripts/calibrate.py` (Python stdlib only) for local aggregation and calibration. +- **File writes:** normal use only prints results to the conversation. Only an explicitly run calibration writes `calibration.json` under the plugin's `scripts/`; the aggregator only reads the votes file you specify. +- **No hooks, no MCP server, no background process, no credentials or user-data collection.** + +## Third-party sources and license + +- The Choice/Noul/Score primitive taxonomy references public Jev (System One, TypeSafe AI) concepts; this plugin is not affiliated with it. +- The rotation-debiasing idea references the AnyJev paper (Nokia Applied Research, Apache-2.0); no code was copied. +- All plugin code is original. MIT License. + +## Test results (2026-10-14, GLM-5.3 as judge) + +**Unit tests** (synthetic data, verifying the math): letter→option mapping, smoothing, and gray-zone abstention all correct; the calibrator reduced ECE from 0.125 to 0.042 on synthetic error data (T=1.45, correct softening direction); a golden-section boundary-update bug was found and fixed during testing. The 0.2.0 NONE outlet and status states passed a nine-path smoke test (majority-NONE correctly triggers `none`, vote splits trigger `clarify`, option renormalization correct). + +**Live-model evaluation** (12 easy + 8 trap items, 3 rotations each, 60 votes total): 60/60 correct, zero vote splits, self-reported judge confidence 72-100 (mean ~97). Position-bias check: across all three rotations judges answered by option content, not position. Consequence: 20/20 correct degenerates temperature fitting ("all-correct data proves nothing"), so **no calibration.json ships**; output honestly reports `calibrated: false` until you calibrate on your own labeled items. + +**Primitive demos**: Choice (email routing, 4 options × 3 rotations) 3/3 unanimous → 0.928; Noul (contract clause, three framings) 0.95/0.98/0.96 → p_true 0.963; Score (5-level rubric) 3/3 → level 4, expected value 3.88. + +**Honest conclusion**: GLM-5.3 as judge is highly consistent on well-posed items; frequency probabilities often hit 1.0 (~0.93 after smoothing); abstention and NONE only fire when judges genuinely split. To profile real failure rates, recalibrate on items your business actually gets wrong. + +## Known limitations + +- Each decision spawns K judge subagents — slower and more session-token-hungry than calling a dedicated decision-model API (you save API cost and privacy) +- Probabilities come from sampling frequency, not token logits; resolution is capped by K (at K=3, non-NONE votes only give 0/⅓/⅔/1) +- Letter readout supports ≤26 options; more requires grouped elimination (see SKILL.md) +- Until calibrated, output reports `calibrated: false` and probabilities skew confident + +## Layout + +``` +.zcode-plugin/plugin.json plugin manifest +skills/jev-sim/SKILL.md decision protocol (core) +scripts/aggregate.py vote aggregation + temperature scaling + NONE/status +scripts/calibrate.py ECE computation + temperature fitting +eval/choice-eval.jsonl 12-item seed evaluation set +``` + +## Versions + +- 0.2.0 — explicit NONE outlet and three-state `status`; option-cap note with grouped sampling; threshold statement (0.6 default is a starting point); audit fields (protocol/threshold/n_samples/none_probability); EN/CN READMEs +- 0.1.0 — Choice/Noul/Score primitives, rotation sampling, temperature-scaling calibration, 20-item evaluation (60/60 votes correct) diff --git a/external_plugins/jev-sim/README_CN.md b/external_plugins/jev-sim/README_CN.md new file mode 100644 index 00000000..c83a2467 --- /dev/null +++ b/external_plugins/jev-sim/README_CN.md @@ -0,0 +1,103 @@ +# jev-sim：会话内决策模型 + +用当前 ZCode 会话里的大模型（GLM）模拟 [Jev](https://typesafe.ai) 式决策模型。把"判断题"变成带概率的类型化决策——不调外部决策模型 API、不需要 API key、数据不出本机。 + +## 三个原语 + +| 原语 | 输入 | 输出 | 典型用途 | +|---|---|---|---| +| Choice | 背景 + 问题 + 选项 | 每个选项的概率 + NONE 出口 | 分类、路由、分支 | +| Noul | 背景 + 命题 | 命题为真的概率 0-1 | 布尔判断、风险筛查 | +| Score | 背景 + 量规 | 等级分布 + 期望分值 | 排序、质量评估 | + +## 工作原理 + +Jev 的底层是把决策变成句子填空，用 token 概率给每个候选打分。会话模型拿不到 token 概率（GLM API 不开放 logprobs），所以用替代机制，全部来自开源决策模型社区的已验证做法： + +1. **字母作答**（letter readout）——判官子代理只回一行 JSON（选项编成 A-Z） +2. **选项轮转**（cyclic shift，AnyJev 的 L0 层）——同一问题换选项顺序问 K=3 遍，消除位置偏差，选票分歧本身就是信息 +3. **对称问句**（Noul 专用）——正问、反问、证据式三视角，消除框架偏差 +4. **频率即概率**——K 张选票平滑归一化成原始概率分布 +5. **温度缩放校准**——拿带标准答案的题拟合温度 T，`p_cal ∝ p_raw^(1/T)`；指示模型普遍过自信，这步把"投票一致性"翻译成"历史上这种把握实际对多少" + +## 输出契约 + +```json +{ + "primitive": "choice", + "protocol": "jev-sim/0.2", + "status": "answer", + "question": "这封邮件路由给谁？", + "decision": { "type": "choice", "choice": "客服-退款", "probability": 0.72 }, + "probabilities": [ + { "option": "客服-退款", "probability": 0.72 }, + { "option": "法务", "probability": 0.14 }, + { "option": "销售", "probability": 0.14 } + ], + "none_probability": 0.02, + "confidence": 0.72, + "threshold": 0.6, + "abstained": false, + "calibrated": true +} +``` + +`status` 三态：**answer**（正常决策）、**none**（NONE 票占多数——没有选项成立，不硬选）、**clarify**（最高概率低于阈值，建议补背景重问或人工处理）。采样/脚本层面的错误按错误原样上报，不伪装成决策。 + +## 使用 + +安装后新建任务，在 composer 里选中 jev-sim 技能，直接给判断题： + +> 背景：发件人李某，正文：你们平台 2 号下单的椅子到货就是断腿的，联系客服三天没人理，今天不退款就投诉…… +> 问题：这封邮件路由给哪个团队？选项：市场 / 客服-退款 / 法务 / 销售 + +**适用**：分类、路由、布尔判断、按量规评分、风险筛查。 +**不适用**：开放式写作、精确符号/字符串查找、算术、能写成确定性规则的判断——这些直接做，不过决策协议。 + +校准（建议首次使用前跑一次）：`python scripts/calibrate.py --results eval/your-results.jsonl`，生成 `scripts/calibration.json` 后所有决策自动套用。自带 `eval/choice-eval.jsonl` 是 12 题种子评测集，方法见 SKILL.md"校准流程"。 + +## 副作用与依赖声明 + +- **网络访问：无。** 全部采样发生在当前 ZCode 会话内，不请求任何外部 API，不发送任何数据。 +- **模型依赖：** 使用会话内置大模型（通过 Agent 工具派出判官子代理）；无需额外账号或密钥。 +- **命令执行：** 运行 `python scripts/aggregate.py`、`scripts/calibrate.py`（仅 Python 标准库）做本地聚合与校准。 +- **文件写入：** 常规使用只把结果输出到对话。仅当主动运行校准时会在插件 `scripts/` 下写 `calibration.json`；聚合脚本只读指定的 votes 文件。 +- **无 hooks、无 MCP server、无后台进程，不收集凭据或用户数据。** + +## 第三方来源与许可 + +- Choice/Noul/Score 原语分类参考 Jev（System One，TypeSafe AI）的公开概念；本插件与其无关联。 +- 选项轮转去偏差思路参考 AnyJev 论文（Nokia Applied Research，Apache-2.0）的 cyclic shift，未复制其代码。 +- 本插件全部代码原创，MIT License。 + +## 测试结果（2026-10-14，GLM-5.3 判官） + +**单元测试**（合成数据，验证脚本数学）：Choice 字母→选项映射、平滑、灰区弃答全部正确；校准器在合成误差数据上把 ECE 从 0.125 降到 0.042（T=1.45，正确软化方向）；期间发现并修复了黄金分割搜索的边界更新 bug。0.2.0 新增的 NONE 出口与 status 三态通过九条路径冒烟测试（多数 NONE 正确触发 none、平票触发 clarify、选项概率重归一正确）。 + +**真模型评测**（12 道易题 + 8 道陷阱题，各 3 轮转，共 60 张选票）：60/60 全对，零选票分裂，判官自报置信度 72-100（均值约 97）。位置偏差检验：三个轮转下判官始终按选项内容作答，未观察到位置偏差。后果：20 题全对使温度拟合退化（"全对数据证明不了什么"），**本插件不预置 calibration.json**，输出如实标注 `calibrated: false`，直到用自己的标注题跑过校准。 + +**三原语功能演示**：Choice（邮件路由，4 选项 3 轮转）3/3 一致 → 0.928；Noul（合同条款，三视角）0.95/0.98/0.96 → p_true 0.963；Score（5 级量规）3/3 一致 → 4=满意，期望分值 3.88。 + +**诚实结论**：GLM-5.3 当判官在明确题上高度一致，频率概率经常是 1.0（平滑后约 0.93）；弃答与 NONE 机制只在判官分裂（真分歧/真难题）时触发。要刻画真实失败率，需要用你自己业务里"会错"的题重新跑校准。 + +## 已知边界 + +- 每次决策要派 K 个判官子代理，比直接调专用决策模型 API 慢、费会话 token（省的是 API 费和隐私） +- 概率来自采样频率而非 token logit，分辨率受轮次 K 限制（K=3 时非 NONE 票只有 0/⅓/⅔/1 四档） +- 字母作答支持 ≤26 个选项，更多需分组初筛（见 SKILL.md） +- 校准没跑过之前，输出标 `calibrated: false`，概率偏自信 + +## 目录 + +``` +.zcode-plugin/plugin.json 插件清单 +skills/jev-sim/SKILL.md 决策协议（核心） +scripts/aggregate.py 选票聚合 + 温度缩放 + NONE/status +scripts/calibrate.py ECE 计算 + 温度拟合 +eval/choice-eval.jsonl 12 题种子评测集 +``` + +## 版本 + +- 0.2.0 —— NONE 显式出口与 `status` 三态输出；选项上限与分组采样说明；阈值声明（默认 0.6 仅为起点）；审计字段（protocol/threshold/n_samples/none_probability）；中英 README +- 0.1.0 —— Choice/Noul/Score 三原语、轮转采样、温度缩放校准、20 题评测（60/60 票正确） diff --git a/external_plugins/jev-sim/eval/choice-eval.jsonl b/external_plugins/jev-sim/eval/choice-eval.jsonl new file mode 100644 index 00000000..0feb4924 --- /dev/null +++ b/external_plugins/jev-sim/eval/choice-eval.jsonl @@ -0,0 +1,12 @@ +{"id": "ev01", "question": "这封邮件应该路由给哪个团队？", "state": "发件人：李某。正文：你们平台2号下单的椅子到货就是断腿的，联系客服三天没人理。今天不给我退款，我就去12315和黑猫投诉，并且找律师发函。订单号SF88231。", "options": ["客服-退款", "法务", "销售", "市场"], "correct": "客服-退款"} +{"id": "ev02", "question": "这段代码是什么语言？", "state": "def flatten(xs):\n out = []\n for x in xs:\n if isinstance(x, list):\n out.extend(flatten(x))\n else:\n out.append(x)\n return out", "options": ["Python", "JavaScript", "Go", "Ruby"], "correct": "Python"} +{"id": "ev03", "question": "这条用户评论的情感倾向是？", "state": "评论原文：真是'太棒了'，买来第二天屏幕就绿了，客服还教我重启大法，学到了学到了。下次一定还来（不会）。", "options": ["正面", "负面", "中性", "无法判断"], "correct": "负面"} +{"id": "ev04", "question": "这个故障该定什么优先级？", "state": "监控告警：23:47 主数据库实例无响应，订单服务全量报错，支付成功率跌到0，持续12分钟，影响所有线上交易。", "options": ["P0-立即响应", "P1-高优处理", "P2-工作时间内处理", "P3-排期处理"], "correct": "P0-立即响应"} +{"id": "ev05", "question": "这段文字是什么语言？", "state": "Trotz der Herausforderungen in den vergangenen Monaten hat das Team eine bemerkenswerte Leistung erbracht, die Anerkennung verdient.", "options": ["德语", "荷兰语", "瑞典语", "丹麦语"], "correct": "德语"} +{"id": "ev06", "question": "这份简历的技术背景最贴近哪个岗位方向？", "state": "技能：Docker、Kubernetes、Terraform、Prometheus、Grafana、Shell、Go 基础。项目：搭建过 200 节点 K8s 集群，落地 CI/CD 流水线，负责服务网格治理。", "options": ["DevOps工程师", "前端工程师", "算法工程师", "数据分析师"], "correct": "DevOps工程师"} +{"id": "ev07", "question": "这段新闻属于哪个频道？", "state": "摘要：某研究团队宣布在超导量子比特的纠错码距离上取得新纪录，逻辑错误率随码距增加实现了指数级下降，相关论文发表于 Nature。", "options": ["科技", "财经", "体育", "娱乐"], "correct": "科技"} +{"id": "ev08", "question": "这段代码的主要缺陷类型是？", "state": "function total(items) {\n let sum = 0;\n for (let i = 0; i <= items.length; i++) {\n sum += items[i].price;\n }\n return sum;\n}", "options": ["边界错误off-by-one", "空指针风险", "死循环", "类型错误"], "correct": "边界错误off-by-one"} +{"id": "ev09", "question": "这封邮件的核心意图是什么？", "state": "发件人：某公司采购部王某。正文：看到贵司展会的工业相机方案，我们要采购30台用于产线质检，请报含税单价、货期和批量折扣，发到这个邮箱即可。", "options": ["销售-询价", "客服-售后", "人事-招聘", "合作-商务"], "correct": "销售-询价"} +{"id": "ev10", "question": "下列哪种动物是哺乳动物？", "state": "（常识判断）", "options": ["海豚", "鲨鱼", "企鹅", "鳄鱼"], "correct": "海豚"} +{"id": "ev11", "question": "这条消息应该路由给哪个处理组？", "state": "用户App内消息：上周买的落地灯罩运损裂了一条缝，灯体没事。不用退货吧？能不能只补发一个灯罩，或者我留着用你们补我差价也行。", "options": ["售后-补发换货", "客服-退款", "物流-理赔", "销售-续购"], "correct": "售后-补发换货"} +{"id": "ev12", "question": "光年是什么类型的单位？", "state": "（常识判断）", "options": ["距离单位", "时间单位", "速度单位", "亮度单位"], "correct": "距离单位"} diff --git a/external_plugins/jev-sim/scripts/aggregate.py b/external_plugins/jev-sim/scripts/aggregate.py new file mode 100644 index 00000000..20eda5c5 --- /dev/null +++ b/external_plugins/jev-sim/scripts/aggregate.py @@ -0,0 +1,219 @@ +#!/usr/bin/env python3 +"""jev-sim 聚合器：选票 → 概率分布 → 温度缩放校准 → 决策 JSON。 + +用法: + python aggregate.py --input votes.json [--threshold 0.6] + +输出带 status 三态：answer（正常决策）/ none（NONE 票占多数，无选项成立）/ +clarify（最高概率低于阈值）。choice/score 的 letter 可以是 "NONE"（v0.2 起）。 + +votes.json 支持三种原语: + +choice / score（score 的 options 即量规等级，可加 "values" 数组给每级数值）: +{ + "primitive": "choice", + "question": "…", + "options": ["原选项1", "原选项2", "原选项3"], + "votes": [ + {"rotation": 0, "order": ["原选项1", "原选项2", "原选项3"], "letter": "B"}, + {"rotation": 1, "order": ["原选项2", "原选项3", "原选项1"], "letter": "A"} + ] +} + +noul: +{ + "primitive": "noul", + "proposition": "…", + "estimates": [ + {"framing": "positive", "p_true": 0.85}, + {"framing": "reversed", "p_true": 0.10}, + {"framing": "evidence", "p_true": 0.80} + ] +} +reversed 框架下 p_true 已经是"命题为真"的口径（调用方负责 1-x 反转），这里直接平均。 + +温度缩放: 若本脚本同目录存在 calibration.json ({"temperature": T})， +choice/score 做 p_cal ∝ p_raw^(1/T)，noul 做二值温度缩放。 +""" +import argparse +import json +import math +import os +import sys + +LETTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" +ALPHA = 0.08 # 平滑系数：计数→概率时混入均匀分布，避免 0/1 硬概率令校准失真 +PROTOCOL = "jev-sim/0.2" + + +def load_temperature(): + cal_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "calibration.json") + if os.path.exists(cal_path): + with open(cal_path, encoding="utf-8") as f: + t = float(json.load(f).get("temperature", 1.0)) + if t > 0: + return t, True + return 1.0, False + + +def temper(probs, t): + """多项温度缩放 p^(1/T) 后归一化。""" + if t == 1.0: + return probs + raw = [max(p, 1e-12) ** (1.0 / t) for p in probs] + s = sum(raw) + return [r / s for r in raw] + + +def temper_binary(p, t): + """二值温度缩放。""" + if t == 1.0: + return p + a = max(p, 1e-12) ** (1.0 / t) + b = max(1.0 - p, 1e-12) ** (1.0 / t) + return a / (a + b) + + +def agg_choice(data, threshold, calibrated): + options = data["options"] + n = len(options) + counts = {o: 0 for o in options} + none_votes = 0 + samples = [] + for v in data["votes"]: + order = v["order"] + letter = str(v["letter"]).strip().upper() + if letter == "NONE": + none_votes += 1 + samples.append({"rotation": v.get("rotation"), "vote": "NONE"}) + continue + idx = LETTERS.index(letter) + picked = order[idx] + counts[picked] += 1 + samples.append({"rotation": v.get("rotation"), "vote": picked}) + k = len(data["votes"]) + # NONE 作为伪选项进入同一平滑空间竞争（多数票语义）；选项概率在排除 NONE 后重归一报告 + denom = k + ALPHA * (n + 1) + p_full = [(counts[o] + ALPHA) / denom for o in options] + p_none = (none_votes + ALPHA) / denom + t, cal = load_temperature() + if cal and t != 1.0: + vec = temper(p_full + [p_none], t) + p_full, p_none = vec[:-1], vec[-1] + mass_opt = 1.0 - p_none + p_cond = [p / mass_opt for p in p_full] if mass_opt > 1e-9 else [1.0 / n] * n + best = max(range(n), key=lambda i: p_full[i]) + conf = p_cond[best] + if none_votes > 0 and p_none > p_full[best]: + status = "none" + elif conf < threshold: + status = "clarify" + else: + status = "answer" + out = { + "primitive": "choice", + "protocol": PROTOCOL, + "status": status, + "question": data.get("question", ""), + "decision": { + "type": "choice", + "choice": None if status == "none" else options[best], + "none": status == "none", + "probability": round(conf, 4), + }, + "probabilities": [ + {"option": o, "probability": round(p, 4)} for o, p in sorted( + zip(options, p_cond), key=lambda x: -x[1]) + ], + "none_probability": round(p_none, 4), + "confidence": round(conf, 4), + "threshold": threshold, + "abstained": status != "answer", + "none_votes": none_votes, + "n_samples": k, + "samples": samples, + "raw_vote_counts": counts, + "calibrated": calibrated and cal, + "temperature": t if cal else None, + } + if status == "none": + out["recommendation"] = "NONE 票占多数（%d/%d）：无选项成立，考虑改问题或补选项" % (none_votes, k) + elif status == "clarify": + out["recommendation"] = "低于阈值 %.2f，建议人工处理或补充背景后重问" % threshold + return out + + +def agg_score(data, threshold, calibrated): + out = agg_choice(data, 0.0, calibrated) + out["primitive"] = "score" + options = data["options"] + probs = [d["probability"] for d in out["probabilities"]] + pmap = {d["option"]: d["probability"] for d in out["probabilities"]} + values = data.get("values", list(range(1, len(options) + 1))) + vmap = {o: values[i] for i, o in enumerate(options)} + expected = sum(pmap[o] * vmap[o] for o in options) + top = out["probabilities"][0] + out["decision"] = { + "type": "score", + "level": None if out["status"] == "none" else top["option"], + "none": out["status"] == "none", + "probability": top["probability"], + "expected_value": round(expected, 3), + } + return out + + +def agg_noul(data, threshold, calibrated): + ests = [float(e["p_true"]) for e in data["estimates"]] + p_raw = sum(ests) / len(ests) + t, cal = load_temperature() + p_cal = temper_binary(p_raw, t) if cal else p_raw + abstained = (p_cal < threshold) and (1.0 - p_cal < threshold) + out = { + "primitive": "noul", + "protocol": PROTOCOL, + "status": "clarify" if abstained else "answer", + "proposition": data.get("proposition", ""), + "decision": { + "type": "noul", + "holds": p_cal >= 0.5, + "p_true": round(p_cal, 4), + }, + "estimates": data["estimates"], + "confidence": round(max(p_cal, 1 - p_cal), 4), + "threshold": threshold, + "abstained": abstained, + "n_samples": len(ests), + "calibrated": calibrated and cal, + "temperature": t if cal else None, + } + if abstained: + out["recommendation"] = "概率落在 [%.2f, %.2f] 灰区，建议人工处理" % (1 - threshold, threshold) + return out + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--input", required=True, help="votes JSON 文件路径，或 '-' 从 stdin 读") + ap.add_argument("--threshold", type=float, default=0.6, help="弃答阈值，默认 0.6") + ap.add_argument("--uncalibrated", action="store_true", help="强制不做温度缩放（调试用）") + args = ap.parse_args() + + raw = sys.stdin.read() if args.input == "-" else open(args.input, encoding="utf-8").read() + data = json.loads(raw) + prim = data.get("primitive", "choice") + + if prim == "noul": + out = agg_noul(data, args.threshold, not args.uncalibrated) + elif prim == "score": + out = agg_score(data, args.threshold, not args.uncalibrated) + else: + out = agg_choice(data, args.threshold, not args.uncalibrated) + + print(json.dumps(out, ensure_ascii=False, indent=2)) + if not out.get("calibrated"): + print("注意：未应用校准（无 calibration.json 或被 --uncalibrated 跳过），概率偏自信。", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/external_plugins/jev-sim/scripts/calibrate.py b/external_plugins/jev-sim/scripts/calibrate.py new file mode 100644 index 00000000..bd94f215 --- /dev/null +++ b/external_plugins/jev-sim/scripts/calibrate.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +"""jev-sim 校准器：拿带标准答案的采样结果拟合温度，写入 calibration.json。 + +用法: + python calibrate.py --results results.jsonl [--out calibration.json] [--alpha 0.08] + +results.jsonl 每行: + {"id": "ev01", "correct": "客服-退款", "probs": {"客服-退款": 1.0, "法务": 0.0, ...}} +probs 是 aggregate 前的原始频率概率（计票/K），correct 是正确选项原文。 + +输出指标: + accuracy argmax 命中率 + logloss 原始概率的 NLL + ece 10 桶校准误差（置信度=最高概率 vs 实际正确率） + temperature 拟合出的 T（黄金分割最小化平滑 NLL） + +拟合前对 probs 做 label smoothing（默认 alpha=0.08）， +否则全票通过的正确项会把 T 拉向 0（过锐）。 +""" +import argparse +import json +import math +import os +import sys +from datetime import datetime + +SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) + + +def safe_path(root, *parts): + p = os.path.realpath(os.path.join(root, *parts)) + r = os.path.realpath(root) + if os.path.commonpath([r, p]) != r: + raise ValueError("path escapes allowed dir: %s" % p) + return p + + +def smooth(probs, alpha): + items = list(probs.items()) + n = len(items) + return {k: (1 - alpha) * v + alpha / n for k, v in items} + + +def nll(items, t): + """items: [(probs_dict, correct)]，温度 t 下的平滑 NLL。""" + total = 0.0 + for probs, correct in items: + raw = [max(p, 1e-12) ** (1.0 / t) for p in probs.values()] + s = sum(raw) + pc = 0.0 + for (k, _), r in zip(probs.items(), raw): + if k == correct: + pc = r / s + break + total += -math.log(max(pc, 1e-12)) + return total / len(items) + + +def ece(items): + """10 桶 ECE，按最高概率分桶。""" + buckets = [[] for _ in range(10)] + for probs, correct in items: + conf = max(probs.values()) + hit = 1.0 if max(probs, key=probs.get) == correct else 0.0 + buckets[min(int(conf * 10), 9)].append((conf, hit)) + n = len(items) + return sum(len(b) / n * abs(sum(h for _, h in b) / len(b) - sum(c for c, _ in b) / len(b)) + for b in buckets if b) + + +def golden_section(f, lo, hi, tol=1e-5): + gr = (math.sqrt(5) - 1) / 2 + a, b = lo, hi + while abs(b - a) > tol: + c = b - gr * (b - a) + d = a + gr * (b - a) + if f(c) < f(d): + b = d + else: + a = c + return (a + b) / 2 + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--results", required=True) + ap.add_argument("--out", default=None) + ap.add_argument("--alpha", type=float, default=0.08) + args = ap.parse_args() + + items = [] + with open(args.results, encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + r = json.loads(line) + items.append((smooth(r["probs"], args.alpha), r["correct"])) + + acc = sum(1 for p, c in items if max(p, key=p.get) == c) / len(items) + raw_items = [(p, c) for p, c in items] # 已平滑 + e_before = ece(raw_items) + t_fit = golden_section(lambda t: nll(items, t), 0.05, 20.0) + e_after = ece([(temper_all(p, t_fit), c) for p, c in items]) + + out_path = safe_path(SCRIPT_DIR, "calibration.json") if args.out is None else safe_path(os.getcwd(), args.out) + result = { + "temperature": round(t_fit, 4), + "fitted_at": datetime.now().isoformat(timespec="seconds"), + "n_items": len(items), + "accuracy": round(acc, 4), + "ece_before": round(e_before, 4), + "ece_after": round(e_after, 4), + "smoothing_alpha": args.alpha, + } + if out_path: + with open(out_path, "w", encoding="utf-8") as f: + json.dump(result, f, ensure_ascii=False, indent=2) + print(json.dumps(result, ensure_ascii=False, indent=2)) + if acc == 1.0: + print("警告：全部答对，温度拟合意义有限——评测集太简单，请加更有区分度的题。", file=sys.stderr) + + +def temper_all(probs, t): + raw = [max(p, 1e-12) ** (1.0 / t) for p in probs.values()] + s = sum(raw) + return {k: r / s for r, (k, _) in zip(raw, probs.items())} + + +if __name__ == "__main__": + main() diff --git a/external_plugins/jev-sim/skills/jev-sim/SKILL.md b/external_plugins/jev-sim/skills/jev-sim/SKILL.md new file mode 100644 index 00000000..a87cee34 --- /dev/null +++ b/external_plugins/jev-sim/skills/jev-sim/SKILL.md @@ -0,0 +1,138 @@ +--- +name: jev-sim +description: 用会话内大模型模拟 Jev 决策模型：把分类、路由、布尔判断、评分变成类型化决策，输出带校准概率的 JSON，有 NONE 出口，不确定时弃答。当用户要"帮我分类/路由/评分/判断真假/风险筛查"，或输入 /jev、说"决策判断"时使用。开放式写作、精确查找、算术、能写成确定性规则的判断不要用它，直接做。 +--- + +# jev-sim：会话内决策模型 + +把"判断题"从聊天变成机器可读的决策。模仿 Jev 的三个原语： + +- **Choice**：从候选里选一个，返回每个选项的概率（分类、路由、分支） +- **Noul**：给一个命题，返回它为真的概率 0-1（布尔判断、风险筛查） +- **Score**：按等级量规（rubric）打分，返回各等级分布（排序、质量判断） + +核心机制（没有 token 概率时的替代方案）：**字母作答 + 选项轮转消位置偏差 + 对称问句消框架偏差 + 多次采样频率估概率 + 温度缩放校准**。 + +## 总流程（四步） + +1. **类型化**：把用户请求解析成 `{state, question, options}`（Choice）、`{state, proposition}`（Noul）或 `{state, question, rubric}`（Score）。state 是背景材料原文，不要自己概括后塞进问题——判官要读原文。 +2. **采样**：按下面的原语协议，用 Agent 工具并行派出多个判官子代理（subagent_type 用 general-purpose），每个判官只回一行 JSON。 +3. **聚合**：把选票写进 votes 文件，跑 `scripts/aggregate.py` 得到概率分布和决策。 +4. **输出**：按"输出契约"给用户 JSON；`none`（无选项成立）或 `clarify`（置信不足）时建议补充背景重问或升级人工。 + +## Choice 协议 + +默认 **K=3 次轮转采样**（选项多或要求高时可加到 5）。 + +轮转方法（循环左移，照抄 AnyJev 的 cyclic shift）：n 个选项、第 i 轮（i=0..K-1）左移 `round(i*n/K)` 位。例：4 选项 3 轮 → 位移 0、1、3→即 [A,B,C,D]、[B,C,D,A]、[D,A,B,C]。**每次轮转后字母按新顺序重新编 A、B、C…**，聚合时按 order 数组映射回原选项。 + +选项上限：字母作答最多支持 26 个选项（A-Z）。更多时分组进行：每组 ≤26 个初筛，每组前 1-2 名进入决赛轮，在结果里注明"分组采样"。 + +判官提示词模板（每个采样一次，一字不差地填入 Agent 工具的 prompt，不要附加别的上下文）： + +``` +你是决策判官，只做判断，不聊天，不解释。 +严格只输出一行 JSON：{"answer":"<一个字母>","confidence":<0-100整数>} + +【背景】 +{state} + +【问题】 +{question} + +【选项】 +A. {选项1} +B. {选项2} +C. {选项3} +... + +从选项中选唯一最可能的。answer 只能是上面出现的字母；若所有选项都不成立，answer 输出 "NONE"。 +``` + +把 K 个采样并行派出（一条消息里多个 Agent 调用）。收齐后写 votes 文件： + +```json +{ + "primitive": "choice", + "question": "…", + "state": "…（可省）", + "options": ["原选项1", "原选项2", "…"], + "votes": [ + {"rotation": 0, "order": ["原选项按轮转0的顺序"], "letter": "B"}, + {"rotation": 1, "order": ["…"], "letter": "A"}, + {"rotation": 2, "order": ["…"], "letter": "B"} + ] +} +``` + +`letter` 正常是 A-Z；判官答了 NONE 时写字符串 `"NONE"`。NONE 是显式出口：它和最高选项概率直接竞争，NONE 占多数时结果进入 `none` 状态，不再硬选。 + +## Noul 协议 + +布尔判断没有选项顺序可轮转，用**对称问句**造出三个不同视角的样本： + +1. **正向**："命题 P 为真吗？" → TRUE/FALSE +2. **反向**："命题『非 P』为真吗？" → TRUE/FALSE（结果反转回来） +3. **证据式**：先在心里各找一条最强的支持和反对证据，再判 P → TRUE/FALSE + +三个模板都要求只回一行 `{"answer":"TRUE|FALSE","confidence":<0-100>}`。反向样本的 p_true 取 `1 - 输出概率`。三个估计取平均得 p_raw。 + +## Score 协议 + +把量规等级当 Choice 的选项跑（等级=选项，"1=完全不符 … 5=完全符合"），得到各等级分布后报**众数等级 + 期望分值**（Σ 等级×概率）。用户给了评分标准就用用户的，没给就先拟一份量规给用户过目再打分。 + +## 聚合与校准 + +votes 文件写好后执行（脚本在本插件 `scripts/` 下，路径相对本 SKILL.md 是 `../../scripts/`）： + +```bash +python "<插件根>/scripts/aggregate.py" --input votes.json [--threshold 0.6] +``` + +脚本做的事：letter 按 order 映射回原选项（NONE 票单独计）→ 计票 → 频率归一化成 p_raw → 若存在 `scripts/calibration.json` 则做温度缩放 `p_cal ∝ p_raw^(1/T)` → 按阈值判 status（answer/none/clarify）。Noul 输入格式见脚本头部注释。没有 Python 环境时手工聚合：计票/K 即 p_raw，报告保留这个事实。 + +## 输出契约（给用户看的东西） + +```json +{ + "primitive": "choice", + "protocol": "jev-sim/0.2", + "status": "answer", + "question": "这封邮件路由给谁？", + "decision": { "type": "choice", "choice": "客服-退款", "probability": 0.72 }, + "probabilities": [ + { "option": "客服-退款", "probability": 0.72 }, + { "option": "法务", "probability": 0.14 }, + { "option": "销售", "probability": 0.14 } + ], + "confidence": 0.72, + "threshold": 0.6, + "abstained": false, + "samples": [ { "rotation": 0, "vote": "客服-退款" } ], + "calibrated": true +} +``` + +规则： +- `status` 三态：`answer`（正常决策）、`none`（NONE 票占多数，无选项成立）、`clarify`（最高概率低于阈值，置信不足）。采样失败、脚本报错等环境错误按错误原样上报，**不要伪装成决策**。 +- `none` 状态：`decision.choice` 为 null，`probabilities` 仅作参考（意思是"如果硬选会是它"）；要明确告诉用户"没有选项成立，考虑改问题或补选项"。 +- **缺失即未知**：背景材料不足以判断时判官会给 NONE 或低置信，输出侧如实转成 none/clarify，绝不替判官脑补。 +- `confidence` = 最高档概率（校准后）。**它不是模型心情，是投票频率经校准后的数**。 +- 阈值默认 0.60 只是起点值。在自己的评测集上跑过校准后，按"想拦下多少错误 vs 忍受多少弃答"的权衡设定，不存在通用正确值。 +- 附一句人话解读 + 哪个选项跟它竞争。不要长篇解释。 + +## 校准流程（一次性/定期） + +概率可信的前提是拟合过温度。拿 20-50 道带标准答案的历史题，按 Choice 协议采样后整理成 results.jsonl（每行 `{"id","correct","probs":{选项:原始概率}}`），然后： + +```bash +python "<插件根>/scripts/calibrate.py" --results results.jsonl +``` + +脚本算准确率、ECE（校准误差），拟合温度 T 写进 `scripts/calibration.json`，之后 aggregate.py 自动使用。**没有跑过校准时，输出里标 `"calibrated": false` 并提醒用户概率偏自信。** + +## 成本控制 + +- 默认 K=3；重要决策再加轮次，不值得的加采样是烧 token。 +- 批量同类判断（如一批邮件各自路由）时，可以把多个条目拼进同一个判官调用（每条一行 JSON 输出），但同批各条目要用不同轮转相位，并在结果里注明"批量采样"。 +- 判官提示词保持极简——判官不需要知道任务背景之外的事。 diff --git a/marketplace.json b/marketplace.json index c95d1fe8..2a64ca03 100644 --- a/marketplace.json +++ b/marketplace.json @@ -790,7 +790,9 @@ }, { "name": "blender", - "componentTypes": ["ui"], + "componentTypes": [ + "ui" + ], "displayName": "Blender", "displayName_i18n": { "en": "Blender", @@ -811,7 +813,9 @@ }, { "name": "excalidraw", - "componentTypes": ["ui"], + "componentTypes": [ + "ui" + ], "displayName": "Excalidraw", "displayName_i18n": { "en": "Excalidraw", @@ -832,7 +836,9 @@ }, { "name": "openpencil", - "componentTypes": ["ui"], + "componentTypes": [ + "ui" + ], "displayName": "OpenPencil", "displayName_i18n": { "en": "OpenPencil", @@ -853,7 +859,9 @@ }, { "name": "showcase", - "componentTypes": ["ui"], + "componentTypes": [ + "ui" + ], "displayName": "Showcase", "displayName_i18n": { "en": "Showcase", @@ -874,7 +882,9 @@ }, { "name": "disk-cleaner", - "componentTypes": ["ui"], + "componentTypes": [ + "ui" + ], "displayName": "Disk Cleaner", "displayName_i18n": { "en": "Disk Cleaner", @@ -896,7 +906,9 @@ { "name": "photocraft", "source": "./plugins/photocraft", - "componentTypes": ["ui"], + "componentTypes": [ + "ui" + ], "displayName": "PhotoCraft", "displayName_i18n": { "en": "PhotoCraft", @@ -922,7 +934,9 @@ { "name": "vectorcraft", "source": "./plugins/vectorcraft", - "componentTypes": ["ui"], + "componentTypes": [ + "ui" + ], "displayName": "VectorCraft", "displayName_i18n": { "en": "VectorCraft", @@ -948,7 +962,9 @@ { "name": "pdfcraft", "source": "./plugins/pdfcraft", - "componentTypes": ["ui"], + "componentTypes": [ + "ui" + ], "displayName": "PdfCraft", "displayName_i18n": { "en": "PdfCraft", @@ -974,7 +990,9 @@ { "name": "designcraft", "source": "./plugins/designcraft", - "componentTypes": ["ui"], + "componentTypes": [ + "ui" + ], "displayName": "DesignCraft", "displayName_i18n": { "en": "DesignCraft", @@ -1000,7 +1018,9 @@ { "name": "wordcraft", "source": "./plugins/wordcraft", - "componentTypes": ["ui"], + "componentTypes": [ + "ui" + ], "displayName": "WordCraft", "displayName_i18n": { "en": "WordCraft", @@ -1026,7 +1046,9 @@ { "name": "gridcraft", "source": "./plugins/gridcraft", - "componentTypes": ["ui"], + "componentTypes": [ + "ui" + ], "displayName": "GridCraft", "displayName_i18n": { "en": "GridCraft", @@ -1052,7 +1074,9 @@ { "name": "deckcraft", "source": "./plugins/deckcraft", - "componentTypes": ["ui"], + "componentTypes": [ + "ui" + ], "displayName": "DeckCraft", "displayName_i18n": { "en": "DeckCraft", @@ -1078,7 +1102,9 @@ { "name": "cadcraft", "source": "./plugins/cadcraft", - "componentTypes": ["ui"], + "componentTypes": [ + "ui" + ], "displayName": "CADCraft", "displayName_i18n": { "en": "CADCraft", @@ -1104,7 +1130,9 @@ { "name": "soundcraft", "source": "./plugins/soundcraft", - "componentTypes": ["ui"], + "componentTypes": [ + "ui" + ], "displayName": "SoundCraft", "displayName_i18n": { "en": "SoundCraft", @@ -1171,6 +1199,36 @@ "生成一个可 3D 打印的骑士小雕像，导出 STL 文件" ] } + }, + { + "name": "jev-sim", + "source": "./external_plugins/jev-sim", + "displayName": "jev-sim", + "displayName_i18n": { + "en": "jev-sim", + "zh-CN": "决策模拟" + }, + "description": "Jev-style decision simulation on the session model: Choice/Noul/Score typed decisions with rotation debiasing, an explicit NONE outlet, three-state output, and temperature-scaling calibration", + "description_i18n": { + "en": "Jev-style decision simulation on the session model: Choice/Noul/Score typed decisions with rotation debiasing, an explicit NONE outlet, three-state output, and temperature-scaling calibration", + "zh-CN": "用会话内大模型模拟 Jev 决策模型：Choice/Noul/Score 类型化决策，轮转去偏差、NONE 出口、三态输出、温度缩放校准" + }, + "version": "0.2.0", + "author": { + "name": "forrestneo" + }, + "category": "developer-tools", + "keywords": [ + "jev", + "decision-model", + "choice", + "noul", + "score", + "calibration", + "routing", + "classification", + "probability" + ] } ] }